1/*
2*******************************************************************************
3*
4*   Copyright (C) 2003-2013, International Business Machines
5*   Corporation and others.  All Rights Reserved.
6*
7*******************************************************************************
8*   file name:  convtest.cpp
9*   encoding:   US-ASCII
10*   tab size:   8 (not used)
11*   indentation:4
12*
13*   created on: 2003jul15
14*   created by: Markus W. Scherer
15*
16*   Test file for data-driven conversion tests.
17*/
18
19#include "unicode/utypes.h"
20
21#if !UCONFIG_NO_LEGACY_CONVERSION
22/*
23 * Note: Turning off all of convtest.cpp if !UCONFIG_NO_LEGACY_CONVERSION
24 * is slightly unnecessary - it removes tests for Unicode charsets
25 * like UTF-8 that should work.
26 * However, there is no easy way for the test to detect whether a test case
27 * is for a Unicode charset, so it would be difficult to only exclude those.
28 * Also, regular testing of ICU is done with all modules on, therefore
29 * not testing conversion for a custom configuration like this should be ok.
30 */
31
32#include "unicode/ucnv.h"
33#include "unicode/unistr.h"
34#include "unicode/parsepos.h"
35#include "unicode/uniset.h"
36#include "unicode/ustring.h"
37#include "unicode/ures.h"
38#include "convtest.h"
39#include "unicode/tstdtmod.h"
40#include <string.h>
41#include <stdlib.h>
42
43#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
44
45enum {
46    // characters used in test data for callbacks
47    SUB_CB='?',
48    SKIP_CB='0',
49    STOP_CB='.',
50    ESC_CB='&'
51};
52
53ConversionTest::ConversionTest() {
54    UErrorCode errorCode=U_ZERO_ERROR;
55    utf8Cnv=ucnv_open("UTF-8", &errorCode);
56    ucnv_setToUCallBack(utf8Cnv, UCNV_TO_U_CALLBACK_STOP, NULL, NULL, NULL, &errorCode);
57    if(U_FAILURE(errorCode)) {
58        errln("unable to open UTF-8 converter");
59    }
60}
61
62ConversionTest::~ConversionTest() {
63    ucnv_close(utf8Cnv);
64}
65
66void
67ConversionTest::runIndexedTest(int32_t index, UBool exec, const char *&name, char * /*par*/) {
68    if (exec) logln("TestSuite ConversionTest: ");
69    switch (index) {
70#if !UCONFIG_NO_FILE_IO
71        case 0: name="TestToUnicode"; if (exec) TestToUnicode(); break;
72        case 1: name="TestFromUnicode"; if (exec) TestFromUnicode(); break;
73        case 2: name="TestGetUnicodeSet"; if (exec) TestGetUnicodeSet(); break;
74#else
75        case 0:
76        case 1:
77        case 2: name="skip"; break;
78#endif
79        case 3: name="TestGetUnicodeSet2"; if (exec) TestGetUnicodeSet2(); break;
80        default: name=""; break; //needed to end loop
81    }
82}
83
84// test data interface ----------------------------------------------------- ***
85
86void
87ConversionTest::TestToUnicode() {
88    ConversionCase cc;
89    char charset[100], cbopt[4];
90    const char *option;
91    UnicodeString s, unicode;
92    int32_t offsetsLength;
93    UConverterToUCallback callback;
94
95    TestDataModule *dataModule;
96    TestData *testData;
97    const DataMap *testCase;
98    UErrorCode errorCode;
99    int32_t i;
100
101    errorCode=U_ZERO_ERROR;
102    dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
103    if(U_SUCCESS(errorCode)) {
104        testData=dataModule->createTestData("toUnicode", errorCode);
105        if(U_SUCCESS(errorCode)) {
106            for(i=0; testData->nextCase(testCase, errorCode); ++i) {
107                if(U_FAILURE(errorCode)) {
108                    errln("error retrieving conversion/toUnicode test case %d - %s",
109                            i, u_errorName(errorCode));
110                    errorCode=U_ZERO_ERROR;
111                    continue;
112                }
113
114                cc.caseNr=i;
115
116                s=testCase->getString("charset", errorCode);
117                s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
118                cc.charset=charset;
119
120                // BEGIN android-added
121                // To save space, Android does not build full ISO-2022-CN tables.
122                // We skip the TestGetKeywordValuesForLocale for counting available collations.
123                if (strlen(charset) >= 8 &&
124                    strncmp(charset+4, "2022-CN", 4) == 0) {
125                    continue;
126                }
127                // END android-added
128
129                cc.bytes=testCase->getBinary(cc.bytesLength, "bytes", errorCode);
130                unicode=testCase->getString("unicode", errorCode);
131                cc.unicode=unicode.getBuffer();
132                cc.unicodeLength=unicode.length();
133
134                offsetsLength=0;
135                cc.offsets=testCase->getIntVector(offsetsLength, "offsets", errorCode);
136                if(offsetsLength==0) {
137                    cc.offsets=NULL;
138                } else if(offsetsLength!=unicode.length()) {
139                    errln("toUnicode[%d] unicode[%d] and offsets[%d] must have the same length",
140                            i, unicode.length(), offsetsLength);
141                    errorCode=U_ILLEGAL_ARGUMENT_ERROR;
142                }
143
144                cc.finalFlush= 0!=testCase->getInt28("flush", errorCode);
145                cc.fallbacks= 0!=testCase->getInt28("fallbacks", errorCode);
146
147                s=testCase->getString("errorCode", errorCode);
148                if(s==UNICODE_STRING("invalid", 7)) {
149                    cc.outErrorCode=U_INVALID_CHAR_FOUND;
150                } else if(s==UNICODE_STRING("illegal", 7)) {
151                    cc.outErrorCode=U_ILLEGAL_CHAR_FOUND;
152                } else if(s==UNICODE_STRING("truncated", 9)) {
153                    cc.outErrorCode=U_TRUNCATED_CHAR_FOUND;
154                } else if(s==UNICODE_STRING("illesc", 6)) {
155                    cc.outErrorCode=U_ILLEGAL_ESCAPE_SEQUENCE;
156                } else if(s==UNICODE_STRING("unsuppesc", 9)) {
157                    cc.outErrorCode=U_UNSUPPORTED_ESCAPE_SEQUENCE;
158                } else {
159                    cc.outErrorCode=U_ZERO_ERROR;
160                }
161
162                s=testCase->getString("callback", errorCode);
163                s.extract(0, 0x7fffffff, cbopt, sizeof(cbopt), "");
164                cc.cbopt=cbopt;
165                switch(cbopt[0]) {
166                case SUB_CB:
167                    callback=UCNV_TO_U_CALLBACK_SUBSTITUTE;
168                    break;
169                case SKIP_CB:
170                    callback=UCNV_TO_U_CALLBACK_SKIP;
171                    break;
172                case STOP_CB:
173                    callback=UCNV_TO_U_CALLBACK_STOP;
174                    break;
175                case ESC_CB:
176                    callback=UCNV_TO_U_CALLBACK_ESCAPE;
177                    break;
178                default:
179                    callback=NULL;
180                    break;
181                }
182                option=callback==NULL ? cbopt : cbopt+1;
183                if(*option==0) {
184                    option=NULL;
185                }
186
187                cc.invalidChars=testCase->getBinary(cc.invalidLength, "invalidChars", errorCode);
188
189                if(U_FAILURE(errorCode)) {
190                    errln("error parsing conversion/toUnicode test case %d - %s",
191                            i, u_errorName(errorCode));
192                    errorCode=U_ZERO_ERROR;
193                } else {
194                    logln("TestToUnicode[%d] %s", i, charset);
195                    ToUnicodeCase(cc, callback, option);
196                }
197            }
198            delete testData;
199        }
200        delete dataModule;
201    }
202    else {
203        dataerrln("Could not load test conversion data");
204    }
205}
206
207void
208ConversionTest::TestFromUnicode() {
209    ConversionCase cc;
210    char charset[100], cbopt[4];
211    const char *option;
212    UnicodeString s, unicode, invalidUChars;
213    int32_t offsetsLength, index;
214    UConverterFromUCallback callback;
215
216    TestDataModule *dataModule;
217    TestData *testData;
218    const DataMap *testCase;
219    const UChar *p;
220    UErrorCode errorCode;
221    int32_t i, length;
222
223    errorCode=U_ZERO_ERROR;
224    dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
225    if(U_SUCCESS(errorCode)) {
226        testData=dataModule->createTestData("fromUnicode", errorCode);
227        if(U_SUCCESS(errorCode)) {
228            for(i=0; testData->nextCase(testCase, errorCode); ++i) {
229                if(U_FAILURE(errorCode)) {
230                    errln("error retrieving conversion/fromUnicode test case %d - %s",
231                            i, u_errorName(errorCode));
232                    errorCode=U_ZERO_ERROR;
233                    continue;
234                }
235
236                cc.caseNr=i;
237
238                s=testCase->getString("charset", errorCode);
239                s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
240                cc.charset=charset;
241
242                // BEGIN android-added
243                // To save space, Android does not build full ISO-2022-CN tables.
244                // We skip the TestGetKeywordValuesForLocale for counting available collations.
245                if (strlen(charset) >= 8 &&
246                    strncmp(charset+4, "2022-CN", 4) == 0) {
247                    continue;
248                }
249                // END android-added
250
251                unicode=testCase->getString("unicode", errorCode);
252                cc.unicode=unicode.getBuffer();
253                cc.unicodeLength=unicode.length();
254                cc.bytes=testCase->getBinary(cc.bytesLength, "bytes", errorCode);
255
256                offsetsLength=0;
257                cc.offsets=testCase->getIntVector(offsetsLength, "offsets", errorCode);
258                if(offsetsLength==0) {
259                    cc.offsets=NULL;
260                } else if(offsetsLength!=cc.bytesLength) {
261                    errln("fromUnicode[%d] bytes[%d] and offsets[%d] must have the same length",
262                            i, cc.bytesLength, offsetsLength);
263                    errorCode=U_ILLEGAL_ARGUMENT_ERROR;
264                }
265
266                cc.finalFlush= 0!=testCase->getInt28("flush", errorCode);
267                cc.fallbacks= 0!=testCase->getInt28("fallbacks", errorCode);
268
269                s=testCase->getString("errorCode", errorCode);
270                if(s==UNICODE_STRING("invalid", 7)) {
271                    cc.outErrorCode=U_INVALID_CHAR_FOUND;
272                } else if(s==UNICODE_STRING("illegal", 7)) {
273                    cc.outErrorCode=U_ILLEGAL_CHAR_FOUND;
274                } else if(s==UNICODE_STRING("truncated", 9)) {
275                    cc.outErrorCode=U_TRUNCATED_CHAR_FOUND;
276                } else {
277                    cc.outErrorCode=U_ZERO_ERROR;
278                }
279
280                s=testCase->getString("callback", errorCode);
281                cc.setSub=0; // default: no subchar
282
283                if((index=s.indexOf((UChar)0))>0) {
284                    // read NUL-separated subchar first, if any
285                    // copy the subchar from Latin-1 characters
286                    // start after the NUL
287                    p=s.getTerminatedBuffer();
288                    length=index+1;
289                    p+=length;
290                    length=s.length()-length;
291                    if(length<=0 || length>=(int32_t)sizeof(cc.subchar)) {
292                        errorCode=U_ILLEGAL_ARGUMENT_ERROR;
293                    } else {
294                        int32_t j;
295
296                        for(j=0; j<length; ++j) {
297                            cc.subchar[j]=(char)p[j];
298                        }
299                        // NUL-terminate the subchar
300                        cc.subchar[j]=0;
301                        cc.setSub=1;
302                    }
303
304                    // remove the NUL and subchar from s
305                    s.truncate(index);
306                } else if((index=s.indexOf((UChar)0x3d))>0) /* '=' */ {
307                    // read a substitution string, separated by an equal sign
308                    p=s.getBuffer()+index+1;
309                    length=s.length()-(index+1);
310                    if(length<0 || length>=LENGTHOF(cc.subString)) {
311                        errorCode=U_ILLEGAL_ARGUMENT_ERROR;
312                    } else {
313                        u_memcpy(cc.subString, p, length);
314                        // NUL-terminate the subString
315                        cc.subString[length]=0;
316                        cc.setSub=-1;
317                    }
318
319                    // remove the equal sign and subString from s
320                    s.truncate(index);
321                }
322
323                s.extract(0, 0x7fffffff, cbopt, sizeof(cbopt), "");
324                cc.cbopt=cbopt;
325                switch(cbopt[0]) {
326                case SUB_CB:
327                    callback=UCNV_FROM_U_CALLBACK_SUBSTITUTE;
328                    break;
329                case SKIP_CB:
330                    callback=UCNV_FROM_U_CALLBACK_SKIP;
331                    break;
332                case STOP_CB:
333                    callback=UCNV_FROM_U_CALLBACK_STOP;
334                    break;
335                case ESC_CB:
336                    callback=UCNV_FROM_U_CALLBACK_ESCAPE;
337                    break;
338                default:
339                    callback=NULL;
340                    break;
341                }
342                option=callback==NULL ? cbopt : cbopt+1;
343                if(*option==0) {
344                    option=NULL;
345                }
346
347                invalidUChars=testCase->getString("invalidUChars", errorCode);
348                cc.invalidUChars=invalidUChars.getBuffer();
349                cc.invalidLength=invalidUChars.length();
350
351                if(U_FAILURE(errorCode)) {
352                    errln("error parsing conversion/fromUnicode test case %d - %s",
353                            i, u_errorName(errorCode));
354                    errorCode=U_ZERO_ERROR;
355                } else {
356                    logln("TestFromUnicode[%d] %s", i, charset);
357                    FromUnicodeCase(cc, callback, option);
358                }
359            }
360            delete testData;
361        }
362        delete dataModule;
363    }
364    else {
365        dataerrln("Could not load test conversion data");
366    }
367}
368
369static const UChar ellipsis[]={ 0x2e, 0x2e, 0x2e };
370
371void
372ConversionTest::TestGetUnicodeSet() {
373    char charset[100];
374    UnicodeString s, map, mapnot;
375    int32_t which;
376
377    ParsePosition pos;
378    UnicodeSet cnvSet, mapSet, mapnotSet, diffSet;
379    UnicodeSet *cnvSetPtr = &cnvSet;
380    LocalUConverterPointer cnv;
381
382    TestDataModule *dataModule;
383    TestData *testData;
384    const DataMap *testCase;
385    UErrorCode errorCode;
386    int32_t i;
387
388    errorCode=U_ZERO_ERROR;
389    dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
390    if(U_SUCCESS(errorCode)) {
391        testData=dataModule->createTestData("getUnicodeSet", errorCode);
392        if(U_SUCCESS(errorCode)) {
393            for(i=0; testData->nextCase(testCase, errorCode); ++i) {
394                if(U_FAILURE(errorCode)) {
395                    errln("error retrieving conversion/getUnicodeSet test case %d - %s",
396                            i, u_errorName(errorCode));
397                    errorCode=U_ZERO_ERROR;
398                    continue;
399                }
400
401                s=testCase->getString("charset", errorCode);
402                s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
403
404                // BEGIN android-added
405                // To save space, Android does not build full ISO-2022-CN tables.
406                // We skip the TestGetKeywordValuesForLocale for counting available collations.
407                if (strlen(charset) >= 8 &&
408                    strncmp(charset+4, "2022-CN", 4) == 0) {
409                    continue;
410                }
411                // END android-added
412
413                map=testCase->getString("map", errorCode);
414                mapnot=testCase->getString("mapnot", errorCode);
415
416                which=testCase->getInt28("which", errorCode);
417
418                if(U_FAILURE(errorCode)) {
419                    errln("error parsing conversion/getUnicodeSet test case %d - %s",
420                            i, u_errorName(errorCode));
421                    errorCode=U_ZERO_ERROR;
422                    continue;
423                }
424
425                // test this test case
426                mapSet.clear();
427                mapnotSet.clear();
428
429                pos.setIndex(0);
430                mapSet.applyPattern(map, pos, 0, NULL, errorCode);
431                if(U_FAILURE(errorCode) || pos.getIndex()!=map.length()) {
432                    errln("error creating the map set for conversion/getUnicodeSet test case %d - %s\n"
433                          "    error index %d  index %d  U+%04x",
434                            i, u_errorName(errorCode), pos.getErrorIndex(), pos.getIndex(), map.char32At(pos.getIndex()));
435                    errorCode=U_ZERO_ERROR;
436                    continue;
437                }
438
439                pos.setIndex(0);
440                mapnotSet.applyPattern(mapnot, pos, 0, NULL, errorCode);
441                if(U_FAILURE(errorCode) || pos.getIndex()!=mapnot.length()) {
442                    errln("error creating the mapnot set for conversion/getUnicodeSet test case %d - %s\n"
443                          "    error index %d  index %d  U+%04x",
444                            i, u_errorName(errorCode), pos.getErrorIndex(), pos.getIndex(), mapnot.char32At(pos.getIndex()));
445                    errorCode=U_ZERO_ERROR;
446                    continue;
447                }
448
449                logln("TestGetUnicodeSet[%d] %s", i, charset);
450
451                cnv.adoptInstead(cnv_open(charset, errorCode));
452                if(U_FAILURE(errorCode)) {
453                    errcheckln(errorCode, "error opening \"%s\" for conversion/getUnicodeSet test case %d - %s",
454                            charset, i, u_errorName(errorCode));
455                    errorCode=U_ZERO_ERROR;
456                    continue;
457                }
458
459                ucnv_getUnicodeSet(cnv.getAlias(), cnvSetPtr->toUSet(), (UConverterUnicodeSet)which, &errorCode);
460
461                if(U_FAILURE(errorCode)) {
462                    errln("error in ucnv_getUnicodeSet(\"%s\") for conversion/getUnicodeSet test case %d - %s",
463                            charset, i, u_errorName(errorCode));
464                    errorCode=U_ZERO_ERROR;
465                    continue;
466                }
467
468                // are there items that must be in cnvSet but are not?
469                (diffSet=mapSet).removeAll(cnvSet);
470                if(!diffSet.isEmpty()) {
471                    diffSet.toPattern(s, TRUE);
472                    if(s.length()>100) {
473                        s.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis));
474                    }
475                    errln("error: ucnv_getUnicodeSet(\"%s\") is missing items - conversion/getUnicodeSet test case %d",
476                            charset, i);
477                    errln(s);
478                }
479
480                // are there items that must not be in cnvSet but are?
481                (diffSet=mapnotSet).retainAll(cnvSet);
482                if(!diffSet.isEmpty()) {
483                    diffSet.toPattern(s, TRUE);
484                    if(s.length()>100) {
485                        s.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis));
486                    }
487                    errln("error: ucnv_getUnicodeSet(\"%s\") contains unexpected items - conversion/getUnicodeSet test case %d",
488                            charset, i);
489                    errln(s);
490                }
491            }
492            delete testData;
493        }
494        delete dataModule;
495    }
496    else {
497        dataerrln("Could not load test conversion data");
498    }
499}
500
501U_CDECL_BEGIN
502static void U_CALLCONV
503getUnicodeSetCallback(const void *context,
504                      UConverterFromUnicodeArgs * /*fromUArgs*/,
505                      const UChar* /*codeUnits*/,
506                      int32_t /*length*/,
507                      UChar32 codePoint,
508                      UConverterCallbackReason reason,
509                      UErrorCode *pErrorCode) {
510    if(reason<=UCNV_IRREGULAR) {
511        ((UnicodeSet *)context)->remove(codePoint);  // the converter cannot convert this code point
512        *pErrorCode=U_ZERO_ERROR;                    // skip
513    }  // else ignore the reset, close and clone calls.
514}
515U_CDECL_END
516
517// Compare ucnv_getUnicodeSet() with the set of characters that can be converted.
518void
519ConversionTest::TestGetUnicodeSet2() {
520    // Build a string with all code points.
521    UChar32 cpLimit;
522    int32_t s0Length;
523    if(quick) {
524        cpLimit=s0Length=0x10000;  // BMP only
525    } else {
526        cpLimit=0x110000;
527        s0Length=0x10000+0x200000;  // BMP + surrogate pairs
528    }
529    UChar *s0=new UChar[s0Length];
530    if(s0==NULL) {
531        return;
532    }
533    UChar *s=s0;
534    UChar32 c;
535    UChar c2;
536    // low BMP
537    for(c=0; c<=0xd7ff; ++c) {
538        *s++=(UChar)c;
539    }
540    // trail surrogates
541    for(c=0xdc00; c<=0xdfff; ++c) {
542        *s++=(UChar)c;
543    }
544    // lead surrogates
545    // (after trails so that there is not even one surrogate pair in between)
546    for(c=0xd800; c<=0xdbff; ++c) {
547        *s++=(UChar)c;
548    }
549    // high BMP
550    for(c=0xe000; c<=0xffff; ++c) {
551        *s++=(UChar)c;
552    }
553    // supplementary code points = surrogate pairs
554    if(cpLimit==0x110000) {
555        for(c=0xd800; c<=0xdbff; ++c) {
556            for(c2=0xdc00; c2<=0xdfff; ++c2) {
557                *s++=(UChar)c;
558                *s++=c2;
559            }
560        }
561    }
562
563    static const char *const cnvNames[]={
564        "UTF-8",
565        "UTF-7",
566        "UTF-16",
567        "US-ASCII",
568        "ISO-8859-1",
569        "windows-1252",
570        "Shift-JIS",
571        "ibm-1390",  // EBCDIC_STATEFUL table
572        "ibm-16684",  // DBCS-only extension table based on EBCDIC_STATEFUL table
573        "HZ",
574        "ISO-2022-JP",
575        "JIS7",
576        "ISO-2022-CN",
577        "ISO-2022-CN-EXT",
578        "LMBCS"
579    };
580    LocalUConverterPointer cnv;
581    char buffer[1024];
582    int32_t i;
583    for(i=0; i<LENGTHOF(cnvNames); ++i) {
584        UErrorCode errorCode=U_ZERO_ERROR;
585        cnv.adoptInstead(cnv_open(cnvNames[i], errorCode));
586        if(U_FAILURE(errorCode)) {
587            errcheckln(errorCode, "failed to open converter %s - %s", cnvNames[i], u_errorName(errorCode));
588            continue;
589        }
590        UnicodeSet expected;
591        ucnv_setFromUCallBack(cnv.getAlias(), getUnicodeSetCallback, &expected, NULL, NULL, &errorCode);
592        if(U_FAILURE(errorCode)) {
593            errln("failed to set the callback on converter %s - %s", cnvNames[i], u_errorName(errorCode));
594            continue;
595        }
596        UConverterUnicodeSet which;
597        for(which=UCNV_ROUNDTRIP_SET; which<UCNV_SET_COUNT; which=(UConverterUnicodeSet)((int)which+1)) {
598            if(which==UCNV_ROUNDTRIP_AND_FALLBACK_SET) {
599                ucnv_setFallback(cnv.getAlias(), TRUE);
600            }
601            expected.add(0, cpLimit-1);
602            s=s0;
603            UBool flush;
604            do {
605                char *t=buffer;
606                flush=(UBool)(s==s0+s0Length);
607                ucnv_fromUnicode(cnv.getAlias(), &t, buffer+sizeof(buffer), (const UChar **)&s, s0+s0Length, NULL, flush, &errorCode);
608                if(U_FAILURE(errorCode)) {
609                    if(errorCode==U_BUFFER_OVERFLOW_ERROR) {
610                        errorCode=U_ZERO_ERROR;
611                        continue;
612                    } else {
613                        break;  // unexpected error, should not occur
614                    }
615                }
616            } while(!flush);
617            UnicodeSet set;
618            ucnv_getUnicodeSet(cnv.getAlias(), set.toUSet(), which, &errorCode);
619            if(cpLimit<0x110000) {
620                set.remove(cpLimit, 0x10ffff);
621            }
622            if(which==UCNV_ROUNDTRIP_SET) {
623                // ignore PUA code points because they will be converted even if they
624                // are fallbacks and when other fallbacks are turned off,
625                // but ucnv_getUnicodeSet(UCNV_ROUNDTRIP_SET) delivers true roundtrips
626                expected.remove(0xe000, 0xf8ff);
627                expected.remove(0xf0000, 0xffffd);
628                expected.remove(0x100000, 0x10fffd);
629                set.remove(0xe000, 0xf8ff);
630                set.remove(0xf0000, 0xffffd);
631                set.remove(0x100000, 0x10fffd);
632            }
633            if(set!=expected) {
634                // First try to see if we have different sets because ucnv_getUnicodeSet()
635                // added strings: The above conversion method does not tell us what strings might be convertible.
636                // Remove strings from the set and compare again.
637                // Unfortunately, there are no good, direct set methods for finding out whether there are strings
638                // in the set, nor for enumerating or removing just them.
639                // Intersect all code points with the set. The intersection will not contain strings.
640                UnicodeSet temp(0, 0x10ffff);
641                temp.retainAll(set);
642                set=temp;
643            }
644            if(set!=expected) {
645                UnicodeSet diffSet;
646                UnicodeString out;
647
648                // are there items that must be in the set but are not?
649                (diffSet=expected).removeAll(set);
650                if(!diffSet.isEmpty()) {
651                    diffSet.toPattern(out, TRUE);
652                    if(out.length()>100) {
653                        out.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis));
654                    }
655                    errln("error: ucnv_getUnicodeSet(\"%s\") is missing items - which set: %d",
656                            cnvNames[i], which);
657                    errln(out);
658                }
659
660                // are there items that must not be in the set but are?
661                (diffSet=set).removeAll(expected);
662                if(!diffSet.isEmpty()) {
663                    diffSet.toPattern(out, TRUE);
664                    if(out.length()>100) {
665                        out.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis));
666                    }
667                    errln("error: ucnv_getUnicodeSet(\"%s\") contains unexpected items - which set: %d",
668                            cnvNames[i], which);
669                    errln(out);
670                }
671            }
672        }
673    }
674
675    delete [] s0;
676}
677
678// open testdata or ICU data converter ------------------------------------- ***
679
680UConverter *
681ConversionTest::cnv_open(const char *name, UErrorCode &errorCode) {
682    if(name!=NULL && *name=='+') {
683        // Converter names that start with '+' are ignored in ICU4J tests.
684        ++name;
685    }
686    if(name!=NULL && *name=='*') {
687        /* loadTestData(): set the data directory */
688        return ucnv_openPackage(loadTestData(errorCode), name+1, &errorCode);
689    } else {
690        return ucnv_open(name, &errorCode);
691    }
692}
693
694// output helpers ---------------------------------------------------------- ***
695
696static inline char
697hexDigit(uint8_t digit) {
698    return digit<=9 ? (char)('0'+digit) : (char)('a'-10+digit);
699}
700
701static char *
702printBytes(const uint8_t *bytes, int32_t length, char *out) {
703    uint8_t b;
704
705    if(length>0) {
706        b=*bytes++;
707        --length;
708        *out++=hexDigit((uint8_t)(b>>4));
709        *out++=hexDigit((uint8_t)(b&0xf));
710    }
711
712    while(length>0) {
713        b=*bytes++;
714        --length;
715        *out++=' ';
716        *out++=hexDigit((uint8_t)(b>>4));
717        *out++=hexDigit((uint8_t)(b&0xf));
718    }
719    *out++=0;
720    return out;
721}
722
723static char *
724printUnicode(const UChar *unicode, int32_t length, char *out) {
725    UChar32 c;
726    int32_t i;
727
728    for(i=0; i<length;) {
729        if(i>0) {
730            *out++=' ';
731        }
732        U16_NEXT(unicode, i, length, c);
733        // write 4..6 digits
734        if(c>=0x100000) {
735            *out++='1';
736        }
737        if(c>=0x10000) {
738            *out++=hexDigit((uint8_t)((c>>16)&0xf));
739        }
740        *out++=hexDigit((uint8_t)((c>>12)&0xf));
741        *out++=hexDigit((uint8_t)((c>>8)&0xf));
742        *out++=hexDigit((uint8_t)((c>>4)&0xf));
743        *out++=hexDigit((uint8_t)(c&0xf));
744    }
745    *out++=0;
746    return out;
747}
748
749static char *
750printOffsets(const int32_t *offsets, int32_t length, char *out) {
751    int32_t i, o, d;
752
753    if(offsets==NULL) {
754        length=0;
755    }
756
757    for(i=0; i<length; ++i) {
758        if(i>0) {
759            *out++=' ';
760        }
761        o=offsets[i];
762
763        // print all offsets with 2 characters each (-x, -9..99, xx)
764        if(o<-9) {
765            *out++='-';
766            *out++='x';
767        } else if(o<0) {
768            *out++='-';
769            *out++=(char)('0'-o);
770        } else if(o<=99) {
771            *out++=(d=o/10)==0 ? ' ' : (char)('0'+d);
772            *out++=(char)('0'+o%10);
773        } else /* o>99 */ {
774            *out++='x';
775            *out++='x';
776        }
777    }
778    *out++=0;
779    return out;
780}
781
782// toUnicode test worker functions ----------------------------------------- ***
783
784static int32_t
785stepToUnicode(ConversionCase &cc, UConverter *cnv,
786              UChar *result, int32_t resultCapacity,
787              int32_t *resultOffsets, /* also resultCapacity */
788              int32_t step,
789              UErrorCode *pErrorCode) {
790    const char *source, *sourceLimit, *bytesLimit;
791    UChar *target, *targetLimit, *resultLimit;
792    UBool flush;
793
794    source=(const char *)cc.bytes;
795    target=result;
796    bytesLimit=source+cc.bytesLength;
797    resultLimit=result+resultCapacity;
798
799    if(step>=0) {
800        // call ucnv_toUnicode() with in/out buffers no larger than (step) at a time
801        // move only one buffer (in vs. out) at a time to be extra mean
802        // step==0 performs bulk conversion and generates offsets
803
804        // initialize the partial limits for the loop
805        if(step==0) {
806            // use the entire buffers
807            sourceLimit=bytesLimit;
808            targetLimit=resultLimit;
809            flush=cc.finalFlush;
810        } else {
811            // start with empty partial buffers
812            sourceLimit=source;
813            targetLimit=target;
814            flush=FALSE;
815
816            // output offsets only for bulk conversion
817            resultOffsets=NULL;
818        }
819
820        for(;;) {
821            // resetting the opposite conversion direction must not affect this one
822            ucnv_resetFromUnicode(cnv);
823
824            // convert
825            ucnv_toUnicode(cnv,
826                &target, targetLimit,
827                &source, sourceLimit,
828                resultOffsets,
829                flush, pErrorCode);
830
831            // check pointers and errors
832            if(source>sourceLimit || target>targetLimit) {
833                *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
834                break;
835            } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
836                if(target!=targetLimit) {
837                    // buffer overflow must only be set when the target is filled
838                    *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
839                    break;
840                } else if(targetLimit==resultLimit) {
841                    // not just a partial overflow
842                    break;
843                }
844
845                // the partial target is filled, set a new limit, reset the error and continue
846                targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
847                *pErrorCode=U_ZERO_ERROR;
848            } else if(U_FAILURE(*pErrorCode)) {
849                // some other error occurred, done
850                break;
851            } else {
852                if(source!=sourceLimit) {
853                    // when no error occurs, then the input must be consumed
854                    *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
855                    break;
856                }
857
858                if(sourceLimit==bytesLimit) {
859                    // we are done
860                    break;
861                }
862
863                // the partial conversion succeeded, set a new limit and continue
864                sourceLimit=(bytesLimit-source)>=step ? source+step : bytesLimit;
865                flush=(UBool)(cc.finalFlush && sourceLimit==bytesLimit);
866            }
867        }
868    } else /* step<0 */ {
869        /*
870         * step==-1: call only ucnv_getNextUChar()
871         * otherwise alternate between ucnv_toUnicode() and ucnv_getNextUChar()
872         *   if step==-2 or -3, then give ucnv_toUnicode() the whole remaining input,
873         *   else give it at most (-step-2)/2 bytes
874         */
875        UChar32 c;
876
877        // end the loop by getting an index out of bounds error
878        for(;;) {
879            // resetting the opposite conversion direction must not affect this one
880            ucnv_resetFromUnicode(cnv);
881
882            // convert
883            if((step&1)!=0 /* odd: -1, -3, -5, ... */) {
884                sourceLimit=source; // use sourceLimit not as a real limit
885                                    // but to remember the pre-getNextUChar source pointer
886                c=ucnv_getNextUChar(cnv, &source, bytesLimit, pErrorCode);
887
888                // check pointers and errors
889                if(*pErrorCode==U_INDEX_OUTOFBOUNDS_ERROR) {
890                    if(source!=bytesLimit) {
891                        *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
892                    } else {
893                        *pErrorCode=U_ZERO_ERROR;
894                    }
895                    break;
896                } else if(U_FAILURE(*pErrorCode)) {
897                    break;
898                }
899                // source may not move if c is from previous overflow
900
901                if(target==resultLimit) {
902                    *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
903                    break;
904                }
905                if(c<=0xffff) {
906                    *target++=(UChar)c;
907                } else {
908                    *target++=U16_LEAD(c);
909                    if(target==resultLimit) {
910                        *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
911                        break;
912                    }
913                    *target++=U16_TRAIL(c);
914                }
915
916                // alternate between -n-1 and -n but leave -1 alone
917                if(step<-1) {
918                    ++step;
919                }
920            } else /* step is even */ {
921                // allow only one UChar output
922                targetLimit=target<resultLimit ? target+1 : resultLimit;
923
924                // as with ucnv_getNextUChar(), we always flush (if we go to bytesLimit)
925                // and never output offsets
926                if(step==-2) {
927                    sourceLimit=bytesLimit;
928                } else {
929                    sourceLimit=source+(-step-2)/2;
930                    if(sourceLimit>bytesLimit) {
931                        sourceLimit=bytesLimit;
932                    }
933                }
934
935                ucnv_toUnicode(cnv,
936                    &target, targetLimit,
937                    &source, sourceLimit,
938                    NULL, (UBool)(sourceLimit==bytesLimit), pErrorCode);
939
940                // check pointers and errors
941                if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
942                    if(target!=targetLimit) {
943                        // buffer overflow must only be set when the target is filled
944                        *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
945                        break;
946                    } else if(targetLimit==resultLimit) {
947                        // not just a partial overflow
948                        break;
949                    }
950
951                    // the partial target is filled, set a new limit and continue
952                    *pErrorCode=U_ZERO_ERROR;
953                } else if(U_FAILURE(*pErrorCode)) {
954                    // some other error occurred, done
955                    break;
956                } else {
957                    if(source!=sourceLimit) {
958                        // when no error occurs, then the input must be consumed
959                        *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
960                        break;
961                    }
962
963                    // we are done (flush==TRUE) but we continue, to get the index out of bounds error above
964                }
965
966                --step;
967            }
968        }
969    }
970
971    return (int32_t)(target-result);
972}
973
974UBool
975ConversionTest::ToUnicodeCase(ConversionCase &cc, UConverterToUCallback callback, const char *option) {
976    // open the converter
977    IcuTestErrorCode errorCode(*this, "ToUnicodeCase");
978    LocalUConverterPointer cnv(cnv_open(cc.charset, errorCode));
979    if(errorCode.isFailure()) {
980        errcheckln(errorCode, "toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_open() failed - %s",
981                cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, errorCode.errorName());
982        errorCode.reset();
983        return FALSE;
984    }
985
986    // set the callback
987    if(callback!=NULL) {
988        ucnv_setToUCallBack(cnv.getAlias(), callback, option, NULL, NULL, errorCode);
989        if(U_FAILURE(errorCode)) {
990            errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setToUCallBack() failed - %s",
991                    cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
992            return FALSE;
993        }
994    }
995
996    int32_t resultOffsets[256];
997    UChar result[256];
998    int32_t resultLength;
999    UBool ok;
1000
1001    static const struct {
1002        int32_t step;
1003        const char *name;
1004    } steps[]={
1005        { 0, "bulk" }, // must be first for offsets to be checked
1006        { 1, "step=1" },
1007        { 3, "step=3" },
1008        { 7, "step=7" },
1009        { -1, "getNext" },
1010        { -2, "toU(bulk)+getNext" },
1011        { -3, "getNext+toU(bulk)" },
1012        { -4, "toU(1)+getNext" },
1013        { -5, "getNext+toU(1)" },
1014        { -12, "toU(5)+getNext" },
1015        { -13, "getNext+toU(5)" },
1016    };
1017    int32_t i, step;
1018
1019    ok=TRUE;
1020    for(i=0; i<LENGTHOF(steps) && ok; ++i) {
1021        step=steps[i].step;
1022        if(step<0 && !cc.finalFlush) {
1023            // skip ucnv_getNextUChar() if !finalFlush because
1024            // ucnv_getNextUChar() always implies flush
1025            continue;
1026        }
1027        if(step!=0) {
1028            // bulk test is first, then offsets are not checked any more
1029            cc.offsets=NULL;
1030        }
1031        else {
1032            memset(resultOffsets, -1, LENGTHOF(resultOffsets));
1033        }
1034        memset(result, -1, LENGTHOF(result));
1035        errorCode.reset();
1036        resultLength=stepToUnicode(cc, cnv.getAlias(),
1037                                result, LENGTHOF(result),
1038                                step==0 ? resultOffsets : NULL,
1039                                step, errorCode);
1040        ok=checkToUnicode(
1041                cc, cnv.getAlias(), steps[i].name,
1042                result, resultLength,
1043                cc.offsets!=NULL ? resultOffsets : NULL,
1044                errorCode);
1045        if(errorCode.isFailure() || !cc.finalFlush) {
1046            // reset if an error occurred or we did not flush
1047            // otherwise do nothing to make sure that flushing resets
1048            ucnv_resetToUnicode(cnv.getAlias());
1049        }
1050        if (cc.offsets != NULL && resultOffsets[resultLength] != -1) {
1051            errln("toUnicode[%d](%s) Conversion wrote too much to offsets at index %d",
1052                cc.caseNr, cc.charset, resultLength);
1053        }
1054        if (result[resultLength] != (UChar)-1) {
1055            errln("toUnicode[%d](%s) Conversion wrote too much to result at index %d",
1056                cc.caseNr, cc.charset, resultLength);
1057        }
1058    }
1059
1060    // not a real loop, just a convenience for breaking out of the block
1061    while(ok && cc.finalFlush) {
1062        // test ucnv_toUChars()
1063        memset(result, 0, sizeof(result));
1064
1065        errorCode.reset();
1066        resultLength=ucnv_toUChars(cnv.getAlias(),
1067                        result, LENGTHOF(result),
1068                        (const char *)cc.bytes, cc.bytesLength,
1069                        errorCode);
1070        ok=checkToUnicode(
1071                cc, cnv.getAlias(), "toUChars",
1072                result, resultLength,
1073                NULL,
1074                errorCode);
1075        if(!ok) {
1076            break;
1077        }
1078
1079        // test preflighting
1080        // keep the correct result for simple checking
1081        errorCode.reset();
1082        resultLength=ucnv_toUChars(cnv.getAlias(),
1083                        NULL, 0,
1084                        (const char *)cc.bytes, cc.bytesLength,
1085                        errorCode);
1086        if(errorCode.get()==U_STRING_NOT_TERMINATED_WARNING || errorCode.get()==U_BUFFER_OVERFLOW_ERROR) {
1087            errorCode.reset();
1088        }
1089        ok=checkToUnicode(
1090                cc, cnv.getAlias(), "preflight toUChars",
1091                result, resultLength,
1092                NULL,
1093                errorCode);
1094        break;
1095    }
1096
1097    errorCode.reset();  // all errors have already been reported
1098    return ok;
1099}
1100
1101UBool
1102ConversionTest::checkToUnicode(ConversionCase &cc, UConverter *cnv, const char *name,
1103                               const UChar *result, int32_t resultLength,
1104                               const int32_t *resultOffsets,
1105                               UErrorCode resultErrorCode) {
1106    char resultInvalidChars[8];
1107    int8_t resultInvalidLength;
1108    UErrorCode errorCode;
1109
1110    const char *msg;
1111
1112    // reset the message; NULL will mean "ok"
1113    msg=NULL;
1114
1115    errorCode=U_ZERO_ERROR;
1116    resultInvalidLength=sizeof(resultInvalidChars);
1117    ucnv_getInvalidChars(cnv, resultInvalidChars, &resultInvalidLength, &errorCode);
1118    if(U_FAILURE(errorCode)) {
1119        errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) ucnv_getInvalidChars() failed - %s",
1120                cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, u_errorName(errorCode));
1121        return FALSE;
1122    }
1123
1124    // check everything that might have gone wrong
1125    if(cc.unicodeLength!=resultLength) {
1126        msg="wrong result length";
1127    } else if(0!=u_memcmp(cc.unicode, result, cc.unicodeLength)) {
1128        msg="wrong result string";
1129    } else if(cc.offsets!=NULL && 0!=memcmp(cc.offsets, resultOffsets, cc.unicodeLength*sizeof(*cc.offsets))) {
1130        msg="wrong offsets";
1131    } else if(cc.outErrorCode!=resultErrorCode) {
1132        msg="wrong error code";
1133    } else if(cc.invalidLength!=resultInvalidLength) {
1134        msg="wrong length of last invalid input";
1135    } else if(0!=memcmp(cc.invalidChars, resultInvalidChars, cc.invalidLength)) {
1136        msg="wrong last invalid input";
1137    }
1138
1139    if(msg==NULL) {
1140        return TRUE;
1141    } else {
1142        char buffer[2000]; // one buffer for all strings
1143        char *s, *bytesString, *unicodeString, *resultString,
1144            *offsetsString, *resultOffsetsString,
1145            *invalidCharsString, *resultInvalidCharsString;
1146
1147        bytesString=s=buffer;
1148        s=printBytes(cc.bytes, cc.bytesLength, bytesString);
1149        s=printUnicode(cc.unicode, cc.unicodeLength, unicodeString=s);
1150        s=printUnicode(result, resultLength, resultString=s);
1151        s=printOffsets(cc.offsets, cc.unicodeLength, offsetsString=s);
1152        s=printOffsets(resultOffsets, resultLength, resultOffsetsString=s);
1153        s=printBytes(cc.invalidChars, cc.invalidLength, invalidCharsString=s);
1154        s=printBytes((uint8_t *)resultInvalidChars, resultInvalidLength, resultInvalidCharsString=s);
1155
1156        if((s-buffer)>(int32_t)sizeof(buffer)) {
1157            errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) fatal error: checkToUnicode() test output buffer overflow writing %d chars\n",
1158                    cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, (int)(s-buffer));
1159            exit(1);
1160        }
1161
1162        errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) failed: %s\n"
1163              "  bytes <%s>[%d]\n"
1164              " expected <%s>[%d]\n"
1165              "  result  <%s>[%d]\n"
1166              " offsets         <%s>\n"
1167              "  result offsets <%s>\n"
1168              " error code expected %s got %s\n"
1169              "  invalidChars expected <%s> got <%s>\n",
1170              cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, msg,
1171              bytesString, cc.bytesLength,
1172              unicodeString, cc.unicodeLength,
1173              resultString, resultLength,
1174              offsetsString,
1175              resultOffsetsString,
1176              u_errorName(cc.outErrorCode), u_errorName(resultErrorCode),
1177              invalidCharsString, resultInvalidCharsString);
1178
1179        return FALSE;
1180    }
1181}
1182
1183// fromUnicode test worker functions --------------------------------------- ***
1184
1185static int32_t
1186stepFromUTF8(ConversionCase &cc,
1187             UConverter *utf8Cnv, UConverter *cnv,
1188             char *result, int32_t resultCapacity,
1189             int32_t step,
1190             UErrorCode *pErrorCode) {
1191    const char *source, *sourceLimit, *utf8Limit;
1192    UChar pivotBuffer[32];
1193    UChar *pivotSource, *pivotTarget, *pivotLimit;
1194    char *target, *targetLimit, *resultLimit;
1195    UBool flush;
1196
1197    source=cc.utf8;
1198    pivotSource=pivotTarget=pivotBuffer;
1199    target=result;
1200    utf8Limit=source+cc.utf8Length;
1201    resultLimit=result+resultCapacity;
1202
1203    // call ucnv_convertEx() with in/out buffers no larger than (step) at a time
1204    // move only one buffer (in vs. out) at a time to be extra mean
1205    // step==0 performs bulk conversion
1206
1207    // initialize the partial limits for the loop
1208    if(step==0) {
1209        // use the entire buffers
1210        sourceLimit=utf8Limit;
1211        targetLimit=resultLimit;
1212        flush=cc.finalFlush;
1213
1214        pivotLimit=pivotBuffer+LENGTHOF(pivotBuffer);
1215    } else {
1216        // start with empty partial buffers
1217        sourceLimit=source;
1218        targetLimit=target;
1219        flush=FALSE;
1220
1221        // empty pivot is not allowed, make it of length step
1222        pivotLimit=pivotBuffer+step;
1223    }
1224
1225    for(;;) {
1226        // resetting the opposite conversion direction must not affect this one
1227        ucnv_resetFromUnicode(utf8Cnv);
1228        ucnv_resetToUnicode(cnv);
1229
1230        // convert
1231        ucnv_convertEx(cnv, utf8Cnv,
1232            &target, targetLimit,
1233            &source, sourceLimit,
1234            pivotBuffer, &pivotSource, &pivotTarget, pivotLimit,
1235            FALSE, flush, pErrorCode);
1236
1237        // check pointers and errors
1238        if(source>sourceLimit || target>targetLimit) {
1239            *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1240            break;
1241        } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1242            if(target!=targetLimit) {
1243                // buffer overflow must only be set when the target is filled
1244                *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1245                break;
1246            } else if(targetLimit==resultLimit) {
1247                // not just a partial overflow
1248                break;
1249            }
1250
1251            // the partial target is filled, set a new limit, reset the error and continue
1252            targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
1253            *pErrorCode=U_ZERO_ERROR;
1254        } else if(U_FAILURE(*pErrorCode)) {
1255            if(pivotSource==pivotBuffer) {
1256                // toUnicode error, should not occur
1257                // toUnicode errors are tested in cintltst TestConvertExFromUTF8()
1258                break;
1259            } else {
1260                // fromUnicode error
1261                // some other error occurred, done
1262                break;
1263            }
1264        } else {
1265            if(source!=sourceLimit) {
1266                // when no error occurs, then the input must be consumed
1267                *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1268                break;
1269            }
1270
1271            if(sourceLimit==utf8Limit) {
1272                // we are done
1273                if(*pErrorCode==U_STRING_NOT_TERMINATED_WARNING) {
1274                    // ucnv_convertEx() warns about not terminating the output
1275                    // but ucnv_fromUnicode() does not and so
1276                    // checkFromUnicode() does not expect it
1277                    *pErrorCode=U_ZERO_ERROR;
1278                }
1279                break;
1280            }
1281
1282            // the partial conversion succeeded, set a new limit and continue
1283            sourceLimit=(utf8Limit-source)>=step ? source+step : utf8Limit;
1284            flush=(UBool)(cc.finalFlush && sourceLimit==utf8Limit);
1285        }
1286    }
1287
1288    return (int32_t)(target-result);
1289}
1290
1291static int32_t
1292stepFromUnicode(ConversionCase &cc, UConverter *cnv,
1293                char *result, int32_t resultCapacity,
1294                int32_t *resultOffsets, /* also resultCapacity */
1295                int32_t step,
1296                UErrorCode *pErrorCode) {
1297    const UChar *source, *sourceLimit, *unicodeLimit;
1298    char *target, *targetLimit, *resultLimit;
1299    UBool flush;
1300
1301    source=cc.unicode;
1302    target=result;
1303    unicodeLimit=source+cc.unicodeLength;
1304    resultLimit=result+resultCapacity;
1305
1306    // call ucnv_fromUnicode() with in/out buffers no larger than (step) at a time
1307    // move only one buffer (in vs. out) at a time to be extra mean
1308    // step==0 performs bulk conversion and generates offsets
1309
1310    // initialize the partial limits for the loop
1311    if(step==0) {
1312        // use the entire buffers
1313        sourceLimit=unicodeLimit;
1314        targetLimit=resultLimit;
1315        flush=cc.finalFlush;
1316    } else {
1317        // start with empty partial buffers
1318        sourceLimit=source;
1319        targetLimit=target;
1320        flush=FALSE;
1321
1322        // output offsets only for bulk conversion
1323        resultOffsets=NULL;
1324    }
1325
1326    for(;;) {
1327        // resetting the opposite conversion direction must not affect this one
1328        ucnv_resetToUnicode(cnv);
1329
1330        // convert
1331        ucnv_fromUnicode(cnv,
1332            &target, targetLimit,
1333            &source, sourceLimit,
1334            resultOffsets,
1335            flush, pErrorCode);
1336
1337        // check pointers and errors
1338        if(source>sourceLimit || target>targetLimit) {
1339            *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1340            break;
1341        } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1342            if(target!=targetLimit) {
1343                // buffer overflow must only be set when the target is filled
1344                *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1345                break;
1346            } else if(targetLimit==resultLimit) {
1347                // not just a partial overflow
1348                break;
1349            }
1350
1351            // the partial target is filled, set a new limit, reset the error and continue
1352            targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
1353            *pErrorCode=U_ZERO_ERROR;
1354        } else if(U_FAILURE(*pErrorCode)) {
1355            // some other error occurred, done
1356            break;
1357        } else {
1358            if(source!=sourceLimit) {
1359                // when no error occurs, then the input must be consumed
1360                *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1361                break;
1362            }
1363
1364            if(sourceLimit==unicodeLimit) {
1365                // we are done
1366                break;
1367            }
1368
1369            // the partial conversion succeeded, set a new limit and continue
1370            sourceLimit=(unicodeLimit-source)>=step ? source+step : unicodeLimit;
1371            flush=(UBool)(cc.finalFlush && sourceLimit==unicodeLimit);
1372        }
1373    }
1374
1375    return (int32_t)(target-result);
1376}
1377
1378UBool
1379ConversionTest::FromUnicodeCase(ConversionCase &cc, UConverterFromUCallback callback, const char *option) {
1380    UConverter *cnv;
1381    UErrorCode errorCode;
1382
1383    // open the converter
1384    errorCode=U_ZERO_ERROR;
1385    cnv=cnv_open(cc.charset, errorCode);
1386    if(U_FAILURE(errorCode)) {
1387        errcheckln(errorCode, "fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_open() failed - %s",
1388                cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1389        return FALSE;
1390    }
1391    ucnv_resetToUnicode(utf8Cnv);
1392
1393    // set the callback
1394    if(callback!=NULL) {
1395        ucnv_setFromUCallBack(cnv, callback, option, NULL, NULL, &errorCode);
1396        if(U_FAILURE(errorCode)) {
1397            errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setFromUCallBack() failed - %s",
1398                    cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1399            ucnv_close(cnv);
1400            return FALSE;
1401        }
1402    }
1403
1404    // set the fallbacks flag
1405    // TODO change with Jitterbug 2401, then add a similar call for toUnicode too
1406    ucnv_setFallback(cnv, cc.fallbacks);
1407
1408    // set the subchar
1409    int32_t length;
1410
1411    if(cc.setSub>0) {
1412        length=(int32_t)strlen(cc.subchar);
1413        ucnv_setSubstChars(cnv, cc.subchar, (int8_t)length, &errorCode);
1414        if(U_FAILURE(errorCode)) {
1415            errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setSubstChars() failed - %s",
1416                    cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1417            ucnv_close(cnv);
1418            return FALSE;
1419        }
1420    } else if(cc.setSub<0) {
1421        ucnv_setSubstString(cnv, cc.subString, -1, &errorCode);
1422        if(U_FAILURE(errorCode)) {
1423            errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setSubstString() failed - %s",
1424                    cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1425            ucnv_close(cnv);
1426            return FALSE;
1427        }
1428    }
1429
1430    // convert unicode to utf8
1431    char utf8[256];
1432    cc.utf8=utf8;
1433    u_strToUTF8(utf8, LENGTHOF(utf8), &cc.utf8Length,
1434                cc.unicode, cc.unicodeLength,
1435                &errorCode);
1436    if(U_FAILURE(errorCode)) {
1437        // skip UTF-8 testing of a string with an unpaired surrogate,
1438        // or of one that's too long
1439        // toUnicode errors are tested in cintltst TestConvertExFromUTF8()
1440        cc.utf8Length=-1;
1441    }
1442
1443    int32_t resultOffsets[256];
1444    char result[256];
1445    int32_t resultLength;
1446    UBool ok;
1447
1448    static const struct {
1449        int32_t step;
1450        const char *name, *utf8Name;
1451    } steps[]={
1452        { 0, "bulk",   "utf8" }, // must be first for offsets to be checked
1453        { 1, "step=1", "utf8 step=1" },
1454        { 3, "step=3", "utf8 step=3" },
1455        { 7, "step=7", "utf8 step=7" }
1456    };
1457    int32_t i, step;
1458
1459    ok=TRUE;
1460    for(i=0; i<LENGTHOF(steps) && ok; ++i) {
1461        step=steps[i].step;
1462        memset(resultOffsets, -1, LENGTHOF(resultOffsets));
1463        memset(result, -1, LENGTHOF(result));
1464        errorCode=U_ZERO_ERROR;
1465        resultLength=stepFromUnicode(cc, cnv,
1466                                result, LENGTHOF(result),
1467                                step==0 ? resultOffsets : NULL,
1468                                step, &errorCode);
1469        ok=checkFromUnicode(
1470                cc, cnv, steps[i].name,
1471                (uint8_t *)result, resultLength,
1472                cc.offsets!=NULL ? resultOffsets : NULL,
1473                errorCode);
1474        if(U_FAILURE(errorCode) || !cc.finalFlush) {
1475            // reset if an error occurred or we did not flush
1476            // otherwise do nothing to make sure that flushing resets
1477            ucnv_resetFromUnicode(cnv);
1478        }
1479        if (resultOffsets[resultLength] != -1) {
1480            errln("fromUnicode[%d](%s) Conversion wrote too much to offsets at index %d",
1481                cc.caseNr, cc.charset, resultLength);
1482        }
1483        if (result[resultLength] != (char)-1) {
1484            errln("fromUnicode[%d](%s) Conversion wrote too much to result at index %d",
1485                cc.caseNr, cc.charset, resultLength);
1486        }
1487
1488        // bulk test is first, then offsets are not checked any more
1489        cc.offsets=NULL;
1490
1491        // test direct conversion from UTF-8
1492        if(cc.utf8Length>=0) {
1493            errorCode=U_ZERO_ERROR;
1494            resultLength=stepFromUTF8(cc, utf8Cnv, cnv,
1495                                    result, LENGTHOF(result),
1496                                    step, &errorCode);
1497            ok=checkFromUnicode(
1498                    cc, cnv, steps[i].utf8Name,
1499                    (uint8_t *)result, resultLength,
1500                    NULL,
1501                    errorCode);
1502            if(U_FAILURE(errorCode) || !cc.finalFlush) {
1503                // reset if an error occurred or we did not flush
1504                // otherwise do nothing to make sure that flushing resets
1505                ucnv_resetToUnicode(utf8Cnv);
1506                ucnv_resetFromUnicode(cnv);
1507            }
1508        }
1509    }
1510
1511    // not a real loop, just a convenience for breaking out of the block
1512    while(ok && cc.finalFlush) {
1513        // test ucnv_fromUChars()
1514        memset(result, 0, sizeof(result));
1515
1516        errorCode=U_ZERO_ERROR;
1517        resultLength=ucnv_fromUChars(cnv,
1518                        result, LENGTHOF(result),
1519                        cc.unicode, cc.unicodeLength,
1520                        &errorCode);
1521        ok=checkFromUnicode(
1522                cc, cnv, "fromUChars",
1523                (uint8_t *)result, resultLength,
1524                NULL,
1525                errorCode);
1526        if(!ok) {
1527            break;
1528        }
1529
1530        // test preflighting
1531        // keep the correct result for simple checking
1532        errorCode=U_ZERO_ERROR;
1533        resultLength=ucnv_fromUChars(cnv,
1534                        NULL, 0,
1535                        cc.unicode, cc.unicodeLength,
1536                        &errorCode);
1537        if(errorCode==U_STRING_NOT_TERMINATED_WARNING || errorCode==U_BUFFER_OVERFLOW_ERROR) {
1538            errorCode=U_ZERO_ERROR;
1539        }
1540        ok=checkFromUnicode(
1541                cc, cnv, "preflight fromUChars",
1542                (uint8_t *)result, resultLength,
1543                NULL,
1544                errorCode);
1545        break;
1546    }
1547
1548    ucnv_close(cnv);
1549    return ok;
1550}
1551
1552UBool
1553ConversionTest::checkFromUnicode(ConversionCase &cc, UConverter *cnv, const char *name,
1554                                 const uint8_t *result, int32_t resultLength,
1555                                 const int32_t *resultOffsets,
1556                                 UErrorCode resultErrorCode) {
1557    UChar resultInvalidUChars[8];
1558    int8_t resultInvalidLength;
1559    UErrorCode errorCode;
1560
1561    const char *msg;
1562
1563    // reset the message; NULL will mean "ok"
1564    msg=NULL;
1565
1566    errorCode=U_ZERO_ERROR;
1567    resultInvalidLength=LENGTHOF(resultInvalidUChars);
1568    ucnv_getInvalidUChars(cnv, resultInvalidUChars, &resultInvalidLength, &errorCode);
1569    if(U_FAILURE(errorCode)) {
1570        errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) ucnv_getInvalidUChars() failed - %s",
1571                cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, u_errorName(errorCode));
1572        return FALSE;
1573    }
1574
1575    // check everything that might have gone wrong
1576    if(cc.bytesLength!=resultLength) {
1577        msg="wrong result length";
1578    } else if(0!=memcmp(cc.bytes, result, cc.bytesLength)) {
1579        msg="wrong result string";
1580    } else if(cc.offsets!=NULL && 0!=memcmp(cc.offsets, resultOffsets, cc.bytesLength*sizeof(*cc.offsets))) {
1581        msg="wrong offsets";
1582    } else if(cc.outErrorCode!=resultErrorCode) {
1583        msg="wrong error code";
1584    } else if(cc.invalidLength!=resultInvalidLength) {
1585        msg="wrong length of last invalid input";
1586    } else if(0!=u_memcmp(cc.invalidUChars, resultInvalidUChars, cc.invalidLength)) {
1587        msg="wrong last invalid input";
1588    }
1589
1590    if(msg==NULL) {
1591        return TRUE;
1592    } else {
1593        char buffer[2000]; // one buffer for all strings
1594        char *s, *unicodeString, *bytesString, *resultString,
1595            *offsetsString, *resultOffsetsString,
1596            *invalidCharsString, *resultInvalidUCharsString;
1597
1598        unicodeString=s=buffer;
1599        s=printUnicode(cc.unicode, cc.unicodeLength, unicodeString);
1600        s=printBytes(cc.bytes, cc.bytesLength, bytesString=s);
1601        s=printBytes(result, resultLength, resultString=s);
1602        s=printOffsets(cc.offsets, cc.bytesLength, offsetsString=s);
1603        s=printOffsets(resultOffsets, resultLength, resultOffsetsString=s);
1604        s=printUnicode(cc.invalidUChars, cc.invalidLength, invalidCharsString=s);
1605        s=printUnicode(resultInvalidUChars, resultInvalidLength, resultInvalidUCharsString=s);
1606
1607        if((s-buffer)>(int32_t)sizeof(buffer)) {
1608            errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) fatal error: checkFromUnicode() test output buffer overflow writing %d chars\n",
1609                    cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, (int)(s-buffer));
1610            exit(1);
1611        }
1612
1613        errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) failed: %s\n"
1614              "  unicode <%s>[%d]\n"
1615              " expected <%s>[%d]\n"
1616              "  result  <%s>[%d]\n"
1617              " offsets         <%s>\n"
1618              "  result offsets <%s>\n"
1619              " error code expected %s got %s\n"
1620              "  invalidChars expected <%s> got <%s>\n",
1621              cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, msg,
1622              unicodeString, cc.unicodeLength,
1623              bytesString, cc.bytesLength,
1624              resultString, resultLength,
1625              offsetsString,
1626              resultOffsetsString,
1627              u_errorName(cc.outErrorCode), u_errorName(resultErrorCode),
1628              invalidCharsString, resultInvalidUCharsString);
1629
1630        return FALSE;
1631    }
1632}
1633
1634#endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */
1635