1/*
2**********************************************************************
3*   Copyright (C) 2008-2009, International Business Machines
4*   Corporation and others.  All Rights Reserved.
5**********************************************************************
6*/
7
8#include "unicode/utypes.h"
9#include "unicode/uspoof.h"
10#include "unicode/unorm.h"
11#include "unicode/uchar.h"
12#include "unicode/uniset.h"
13#include "utrie2.h"
14#include "cmemory.h"
15#include "cstring.h"
16#include "udatamem.h"
17#include "umutex.h"
18#include "udataswp.h"
19#include "uassert.h"
20#include "uspoof_impl.h"
21
22#if !UCONFIG_NO_NORMALIZATION
23
24
25U_NAMESPACE_BEGIN
26
27UOBJECT_DEFINE_RTTI_IMPLEMENTATION(SpoofImpl)
28
29SpoofImpl::SpoofImpl(SpoofData *data, UErrorCode &status) :
30    fMagic(0), fSpoofData(NULL), fAllowedCharsSet(NULL) , fAllowedLocales(NULL) {
31    if (U_FAILURE(status)) {
32        return;
33    }
34    fMagic = USPOOF_MAGIC;
35    fSpoofData = data;
36    fChecks = USPOOF_ALL_CHECKS;
37    UnicodeSet *allowedCharsSet = new UnicodeSet(0, 0x10ffff);
38    if (allowedCharsSet == NULL) {
39        status = U_MEMORY_ALLOCATION_ERROR;
40    }
41    allowedCharsSet->freeze();
42    fAllowedCharsSet = allowedCharsSet;
43    fAllowedLocales  = uprv_strdup("");
44}
45
46
47SpoofImpl::SpoofImpl() {
48    fMagic = USPOOF_MAGIC;
49    fSpoofData = NULL;
50    fChecks = USPOOF_ALL_CHECKS;
51    UnicodeSet *allowedCharsSet = new UnicodeSet(0, 0x10ffff);
52    allowedCharsSet->freeze();
53    fAllowedCharsSet = allowedCharsSet;
54    fAllowedLocales  = uprv_strdup("");
55}
56
57
58// Copy Constructor, used by the user level clone() function.
59SpoofImpl::SpoofImpl(const SpoofImpl &src, UErrorCode &status)  :
60    fMagic(0), fSpoofData(NULL), fAllowedCharsSet(NULL) {
61    if (U_FAILURE(status)) {
62        return;
63    }
64    fMagic = src.fMagic;
65    fChecks = src.fChecks;
66    if (src.fSpoofData != NULL) {
67        fSpoofData = src.fSpoofData->addReference();
68    }
69    fCheckMask = src.fCheckMask;
70    fAllowedCharsSet = static_cast<const UnicodeSet *>(src.fAllowedCharsSet->clone());
71    if (fAllowedCharsSet == NULL) {
72        status = U_MEMORY_ALLOCATION_ERROR;
73    }
74    fAllowedLocales = uprv_strdup(src.fAllowedLocales);
75}
76
77SpoofImpl::~SpoofImpl() {
78    fMagic = 0;                // head off application errors by preventing use of
79                               //    of deleted objects.
80    if (fSpoofData != NULL) {
81        fSpoofData->removeReference();   // Will delete if refCount goes to zero.
82    }
83    delete fAllowedCharsSet;
84    uprv_free((void *)fAllowedLocales);
85}
86
87//
88//  Incoming parameter check on Status and the SpoofChecker object
89//    received from the C API.
90//
91const SpoofImpl *SpoofImpl::validateThis(const USpoofChecker *sc, UErrorCode &status) {
92    if (U_FAILURE(status)) {
93        return NULL;
94    }
95    if (sc == NULL) {
96        status = U_ILLEGAL_ARGUMENT_ERROR;
97        return NULL;
98    };
99    SpoofImpl *This = (SpoofImpl *)sc;
100    if (This->fMagic != USPOOF_MAGIC ||
101        This->fSpoofData == NULL) {
102        status = U_INVALID_FORMAT_ERROR;
103        return NULL;
104    }
105    if (!SpoofData::validateDataVersion(This->fSpoofData->fRawData, status)) {
106        return NULL;
107    }
108    return This;
109}
110
111SpoofImpl *SpoofImpl::validateThis(USpoofChecker *sc, UErrorCode &status) {
112    return const_cast<SpoofImpl *>
113        (SpoofImpl::validateThis(const_cast<const USpoofChecker *>(sc), status));
114}
115
116
117
118//--------------------------------------------------------------------------------------
119//
120//  confusableLookup()    This is the heart of the confusable skeleton generation
121//                        implementation.
122//
123//                        Given a source character, produce the corresponding
124//                        replacement character(s)
125//
126//---------------------------------------------------------------------------------------
127int32_t SpoofImpl::confusableLookup(UChar32 inChar, int32_t tableMask, UChar *destBuf) const {
128
129    // Binary search the spoof data key table for the inChar
130    int32_t  *low   = fSpoofData->fCFUKeys;
131    int32_t  *mid   = NULL;
132    int32_t  *limit = low + fSpoofData->fRawData->fCFUKeysSize;
133    UChar32   midc;
134    do {
135        int32_t delta = ((int32_t)(limit-low))/2;
136        mid = low + delta;
137        midc = *mid & 0x1fffff;
138        if (inChar == midc) {
139            goto foundChar;
140        } else if (inChar < midc) {
141            limit = mid;
142        } else {
143            low = mid;
144        }
145    } while (low < limit-1);
146    mid = low;
147    midc = *mid & 0x1fffff;
148    if (inChar != midc) {
149        // Char not found.  It maps to itself.
150        int i = 0;
151        U16_APPEND_UNSAFE(destBuf, i, inChar)
152        return i;
153    }
154  foundChar:
155    int32_t keyFlags = *mid & 0xff000000;
156    if ((keyFlags & tableMask) == 0) {
157        // We found the right key char, but the entry doesn't pertain to the
158        //  table we need.  See if there is an adjacent key that does
159        if (keyFlags & USPOOF_KEY_MULTIPLE_VALUES) {
160            int32_t *altMid;
161            for (altMid = mid-1; (*altMid&0x00ffffff) == inChar; altMid--) {
162                keyFlags = *altMid & 0xff000000;
163                if (keyFlags & tableMask) {
164                    mid = altMid;
165                    goto foundKey;
166                }
167            }
168            for (altMid = mid+1; (*altMid&0x00ffffff) == inChar; altMid++) {
169                keyFlags = *altMid & 0xff000000;
170                if (keyFlags & tableMask) {
171                    mid = altMid;
172                    goto foundKey;
173                }
174            }
175        }
176        // No key entry for this char & table.
177        // The input char maps to itself.
178        int i = 0;
179        U16_APPEND_UNSAFE(destBuf, i, inChar)
180        return i;
181    }
182
183  foundKey:
184    int32_t  stringLen = USPOOF_KEY_LENGTH_FIELD(keyFlags) + 1;
185    int32_t keyTableIndex = (int32_t)(mid - fSpoofData->fCFUKeys);
186
187    // Value is either a UChar  (for strings of length 1) or
188    //                 an index into the string table (for longer strings)
189    uint16_t value = fSpoofData->fCFUValues[keyTableIndex];
190    if (stringLen == 1) {
191        destBuf[0] = value;
192        return 1;
193    }
194
195    // String length of 4 from the above lookup is used for all strings of length >= 4.
196    // For these, get the real length from the string lengths table,
197    //   which maps string table indexes to lengths.
198    //   All strings of the same length are stored contiguously in the string table.
199    //   'value' from the lookup above is the starting index for the desired string.
200
201    int32_t ix;
202    if (stringLen == 4) {
203        int32_t stringLengthsLimit = fSpoofData->fRawData->fCFUStringLengthsSize;
204        for (ix = 0; ix < stringLengthsLimit; ix++) {
205            if (fSpoofData->fCFUStringLengths[ix].fLastString >= value) {
206                stringLen = fSpoofData->fCFUStringLengths[ix].fStrLength;
207                break;
208            }
209        }
210        U_ASSERT(ix < stringLengthsLimit);
211    }
212
213    U_ASSERT(value + stringLen < fSpoofData->fRawData->fCFUStringTableLen);
214    UChar *src = &fSpoofData->fCFUStrings[value];
215    for (ix=0; ix<stringLen; ix++) {
216        destBuf[ix] = src[ix];
217    }
218    return stringLen;
219}
220
221
222//---------------------------------------------------------------------------------------
223//
224//  wholeScriptCheck()
225//
226//      Input text is already normalized to NFKD
227//      Return the set of scripts, each of which can represent something that is
228//             confusable with the input text.  The script of the input text
229//             is included; input consisting of characters from a single script will
230//             always produce a result consisting of a set containing that script.
231//
232//---------------------------------------------------------------------------------------
233void SpoofImpl::wholeScriptCheck(
234    const UChar *text, int32_t length, ScriptSet *result, UErrorCode &status) const {
235
236    int32_t       inputIdx = 0;
237    UChar32       c;
238
239    UTrie2 *table =
240        (fChecks & USPOOF_ANY_CASE) ? fSpoofData->fAnyCaseTrie : fSpoofData->fLowerCaseTrie;
241    result->setAll();
242    while (inputIdx < length) {
243        U16_NEXT(text, inputIdx, length, c);
244        uint32_t index = utrie2_get32(table, c);
245        if (index == 0) {
246            // No confusables in another script for this char.
247            // TODO:  we should change the data to have sets with just the single script
248            //        bit for the script of this char.  Gets rid of this special case.
249            //        Until then, grab the script from the char and intersect it with the set.
250            UScriptCode cpScript = uscript_getScript(c, &status);
251            U_ASSERT(cpScript > USCRIPT_INHERITED);
252            result->intersect(cpScript);
253        } else if (index == 1) {
254            // Script == Common or Inherited.  Nothing to do.
255        } else {
256            result->intersect(fSpoofData->fScriptSets[index]);
257        }
258    }
259}
260
261
262void SpoofImpl::setAllowedLocales(const char *localesList, UErrorCode &status) {
263    UnicodeSet    allowedChars;
264    UnicodeSet    *tmpSet = NULL;
265    const char    *locStart = localesList;
266    const char    *locEnd = NULL;
267    const char    *localesListEnd = localesList + uprv_strlen(localesList);
268    int32_t        localeListCount = 0;   // Number of locales provided by caller.
269
270    // Loop runs once per locale from the localesList, a comma separated list of locales.
271    do {
272        locEnd = uprv_strchr(locStart, ',');
273        if (locEnd == NULL) {
274            locEnd = localesListEnd;
275        }
276        while (*locStart == ' ') {
277            locStart++;
278        }
279        const char *trimmedEnd = locEnd-1;
280        while (trimmedEnd > locStart && *trimmedEnd == ' ') {
281            trimmedEnd--;
282        }
283        if (trimmedEnd <= locStart) {
284            break;
285        }
286        const char *locale = uprv_strndup(locStart, (int32_t)(trimmedEnd + 1 - locStart));
287        localeListCount++;
288
289        // We have one locale from the locales list.
290        // Add the script chars for this locale to the accumulating set of allowed chars.
291        // If the locale is no good, we will be notified back via status.
292        addScriptChars(locale, &allowedChars, status);
293        uprv_free((void *)locale);
294        if (U_FAILURE(status)) {
295            break;
296        }
297        locStart = locEnd + 1;
298    } while (locStart < localesListEnd);
299
300    // If our caller provided an empty list of locales, we disable the allowed characters checking
301    if (localeListCount == 0) {
302        uprv_free((void *)fAllowedLocales);
303        fAllowedLocales = uprv_strdup("");
304        tmpSet = new UnicodeSet(0, 0x10ffff);
305        if (fAllowedLocales == NULL || tmpSet == NULL) {
306            status = U_MEMORY_ALLOCATION_ERROR;
307            return;
308        }
309        tmpSet->freeze();
310        delete fAllowedCharsSet;
311        fAllowedCharsSet = tmpSet;
312        fCheckMask &= ~USPOOF_CHAR_LIMIT;
313        return;
314    }
315
316
317    // Add all common and inherited characters to the set of allowed chars.
318    UnicodeSet tempSet;
319    tempSet.applyIntPropertyValue(UCHAR_SCRIPT, USCRIPT_COMMON, status);
320    allowedChars.addAll(tempSet);
321    tempSet.applyIntPropertyValue(UCHAR_SCRIPT, USCRIPT_INHERITED, status);
322    allowedChars.addAll(tempSet);
323
324    // If anything went wrong, we bail out without changing
325    // the state of the spoof checker.
326    if (U_FAILURE(status)) {
327        return;
328    }
329
330    // Store the updated spoof checker state.
331    tmpSet = static_cast<UnicodeSet *>(allowedChars.clone());
332    const char *tmpLocalesList = uprv_strdup(localesList);
333    if (tmpSet == NULL || tmpLocalesList == NULL) {
334        status = U_MEMORY_ALLOCATION_ERROR;
335        return;
336    }
337    uprv_free((void *)fAllowedLocales);
338    fAllowedLocales = tmpLocalesList;
339    tmpSet->freeze();
340    delete fAllowedCharsSet;
341    fAllowedCharsSet = tmpSet;
342    fCheckMask |= USPOOF_CHAR_LIMIT;
343}
344
345
346const char * SpoofImpl::getAllowedLocales(UErrorCode &/*status*/) {
347    return fAllowedLocales;
348}
349
350
351// Given a locale (a language), add all the characters from all of the scripts used with that language
352// to the allowedChars UnicodeSet
353
354void SpoofImpl::addScriptChars(const char *locale, UnicodeSet *allowedChars, UErrorCode &status) {
355    UScriptCode scripts[30];
356
357    int32_t numScripts = uscript_getCode(locale, scripts, sizeof(scripts)/sizeof(UScriptCode), &status);
358    if (U_FAILURE(status)) {
359        return;
360    }
361    if (status == U_USING_DEFAULT_WARNING) {
362        status = U_ILLEGAL_ARGUMENT_ERROR;
363        return;
364    }
365    UnicodeSet tmpSet;
366    int32_t    i;
367    for (i=0; i<numScripts; i++) {
368        tmpSet.applyIntPropertyValue(UCHAR_SCRIPT, scripts[i], status);
369        allowedChars->addAll(tmpSet);
370    }
371}
372
373
374int32_t SpoofImpl::scriptScan
375        (const UChar *text, int32_t length, int32_t &pos, UErrorCode &status) const {
376    if (U_FAILURE(status)) {
377        return 0;
378    }
379    int32_t       inputIdx = 0;
380    UChar32       c;
381    int32_t       scriptCount = 0;
382    UScriptCode   lastScript = USCRIPT_INVALID_CODE;
383    UScriptCode   sc = USCRIPT_INVALID_CODE;
384    while ((inputIdx < length || length == -1) && scriptCount < 2) {
385        U16_NEXT(text, inputIdx, length, c);
386        if (c == 0 && length == -1) {
387            break;
388        }
389        sc = uscript_getScript(c, &status);
390        if (sc == USCRIPT_COMMON || sc == USCRIPT_INHERITED || sc == USCRIPT_UNKNOWN) {
391            continue;
392        }
393        if (sc != lastScript) {
394           scriptCount++;
395           lastScript = sc;
396        }
397    }
398    if (scriptCount == 2) {
399        pos = inputIdx;
400    }
401    return scriptCount;
402}
403
404
405// Convert a text format hex number.  Utility function used by builder code.  Static.
406// Input: UChar *string text.  Output: a UChar32
407// Input has been pre-checked, and will have no non-hex chars.
408// The number must fall in the code point range of 0..0x10ffff
409// Static Function.
410UChar32 SpoofImpl::ScanHex(const UChar *s, int32_t start, int32_t limit, UErrorCode &status) {
411    if (U_FAILURE(status)) {
412        return 0;
413    }
414    U_ASSERT(limit-start > 0);
415    uint32_t val = 0;
416    int i;
417    for (i=start; i<limit; i++) {
418        int digitVal = s[i] - 0x30;
419        if (digitVal>9) {
420            digitVal = 0xa + (s[i] - 0x41);  // Upper Case 'A'
421        }
422        if (digitVal>15) {
423            digitVal = 0xa + (s[i] - 0x61);  // Lower Case 'a'
424        }
425        U_ASSERT(digitVal <= 0xf);
426        val <<= 4;
427        val += digitVal;
428    }
429    if (val > 0x10ffff) {
430        status = U_PARSE_ERROR;
431        val = 0;
432    }
433    return (UChar32)val;
434}
435
436
437
438//----------------------------------------------------------------------------------------------
439//
440//   class SpoofData Implementation
441//
442//----------------------------------------------------------------------------------------------
443
444
445UBool SpoofData::validateDataVersion(const SpoofDataHeader *rawData, UErrorCode &status) {
446    if (U_FAILURE(status) ||
447        rawData == NULL ||
448        rawData->fMagic != USPOOF_MAGIC ||
449        rawData->fFormatVersion[0] > 1 ||
450        rawData->fFormatVersion[1] > 0) {
451            status = U_INVALID_FORMAT_ERROR;
452            return FALSE;
453    }
454    return TRUE;
455}
456
457//
458//  SpoofData::getDefault() - return a wrapper around the spoof data that is
459//                           baked into the default ICU data.
460//
461SpoofData *SpoofData::getDefault(UErrorCode &status) {
462    // TODO:  Cache it.  Lazy create, keep until cleanup.
463
464    UDataMemory *udm = udata_open(NULL, "cfu", "confusables", &status);
465    if (U_FAILURE(status)) {
466        return NULL;
467    }
468    SpoofData *This = new SpoofData(udm, status);
469    if (U_FAILURE(status)) {
470        delete This;
471        return NULL;
472    }
473    if (This == NULL) {
474        status = U_MEMORY_ALLOCATION_ERROR;
475    }
476    return This;
477}
478
479
480SpoofData::SpoofData(UDataMemory *udm, UErrorCode &status)
481{
482    reset();
483    if (U_FAILURE(status)) {
484        return;
485    }
486    fRawData = reinterpret_cast<SpoofDataHeader *>
487                   ((char *)(udm->pHeader) + udm->pHeader->dataHeader.headerSize);
488    fUDM = udm;
489    validateDataVersion(fRawData, status);
490    initPtrs(status);
491}
492
493
494SpoofData::SpoofData(const void *data, int32_t length, UErrorCode &status)
495{
496    reset();
497    if (U_FAILURE(status)) {
498        return;
499    }
500    if ((size_t)length < sizeof(SpoofDataHeader)) {
501        status = U_INVALID_FORMAT_ERROR;
502        return;
503    }
504    void *ncData = const_cast<void *>(data);
505    fRawData = static_cast<SpoofDataHeader *>(ncData);
506    if (length < fRawData->fLength) {
507        status = U_INVALID_FORMAT_ERROR;
508        return;
509    }
510    validateDataVersion(fRawData, status);
511    initPtrs(status);
512}
513
514
515// Spoof Data constructor for use from data builder.
516//   Initializes a new, empty data area that will be populated later.
517SpoofData::SpoofData(UErrorCode &status) {
518    reset();
519    if (U_FAILURE(status)) {
520        return;
521    }
522    fDataOwned = true;
523    fRefCount = 1;
524
525    // The spoof header should already be sized to be a multiple of 16 bytes.
526    // Just in case it's not, round it up.
527    uint32_t initialSize = (sizeof(SpoofDataHeader) + 15) & ~15;
528    U_ASSERT(initialSize == sizeof(SpoofDataHeader));
529
530    fRawData = static_cast<SpoofDataHeader *>(uprv_malloc(initialSize));
531    fMemLimit = initialSize;
532    if (fRawData == NULL) {
533        status = U_MEMORY_ALLOCATION_ERROR;
534        return;
535    }
536    uprv_memset(fRawData, 0, initialSize);
537
538    fRawData->fMagic = USPOOF_MAGIC;
539    fRawData->fFormatVersion[0] = 1;
540    fRawData->fFormatVersion[1] = 0;
541    fRawData->fFormatVersion[2] = 0;
542    fRawData->fFormatVersion[3] = 0;
543    initPtrs(status);
544}
545
546// reset() - initialize all fields.
547//           Should be updated if any new fields are added.
548//           Called by constructors to put things in a known initial state.
549void SpoofData::reset() {
550   fRawData = NULL;
551   fDataOwned = FALSE;
552   fUDM      = NULL;
553   fMemLimit = 0;
554   fRefCount = 1;
555   fCFUKeys = NULL;
556   fCFUValues = NULL;
557   fCFUStringLengths = NULL;
558   fCFUStrings = NULL;
559   fAnyCaseTrie = NULL;
560   fLowerCaseTrie = NULL;
561   fScriptSets = NULL;
562}
563
564
565//  SpoofData::initPtrs()
566//            Initialize the pointers to the various sections of the raw data.
567//
568//            This function is used both during the Trie building process (multiple
569//            times, as the individual data sections are added), and
570//            during the opening of a Spoof Checker from prebuilt data.
571//
572//            The pointers for non-existent data sections (identified by an offset of 0)
573//            are set to NULL.
574//
575//            Note:  During building the data, adding each new data section
576//            reallocs the raw data area, which likely relocates it, which
577//            in turn requires reinitializing all of the pointers into it, hence
578//            multiple calls to this function during building.
579//
580void SpoofData::initPtrs(UErrorCode &status) {
581    fCFUKeys = NULL;
582    fCFUValues = NULL;
583    fCFUStringLengths = NULL;
584    fCFUStrings = NULL;
585    if (U_FAILURE(status)) {
586        return;
587    }
588    if (fRawData->fCFUKeys != 0) {
589        fCFUKeys = (int32_t *)((char *)fRawData + fRawData->fCFUKeys);
590    }
591    if (fRawData->fCFUStringIndex != 0) {
592        fCFUValues = (uint16_t *)((char *)fRawData + fRawData->fCFUStringIndex);
593    }
594    if (fRawData->fCFUStringLengths != 0) {
595        fCFUStringLengths = (SpoofStringLengthsElement *)((char *)fRawData + fRawData->fCFUStringLengths);
596    }
597    if (fRawData->fCFUStringTable != 0) {
598        fCFUStrings = (UChar *)((char *)fRawData + fRawData->fCFUStringTable);
599    }
600
601    if (fAnyCaseTrie ==  NULL && fRawData->fAnyCaseTrie != 0) {
602        fAnyCaseTrie = utrie2_openFromSerialized(UTRIE2_16_VALUE_BITS,
603            (char *)fRawData + fRawData->fAnyCaseTrie, fRawData->fAnyCaseTrieLength, NULL, &status);
604    }
605    if (fLowerCaseTrie ==  NULL && fRawData->fLowerCaseTrie != 0) {
606        fLowerCaseTrie = utrie2_openFromSerialized(UTRIE2_16_VALUE_BITS,
607            (char *)fRawData + fRawData->fLowerCaseTrie, fRawData->fLowerCaseTrieLength, NULL, &status);
608    }
609
610    if (fRawData->fScriptSets != 0) {
611        fScriptSets = (ScriptSet *)((char *)fRawData + fRawData->fScriptSets);
612    }
613}
614
615
616SpoofData::~SpoofData() {
617    utrie2_close(fAnyCaseTrie);
618    fAnyCaseTrie = NULL;
619    utrie2_close(fLowerCaseTrie);
620    fLowerCaseTrie = NULL;
621    if (fDataOwned) {
622        uprv_free(fRawData);
623    }
624    fRawData = NULL;
625    if (fUDM != NULL) {
626        udata_close(fUDM);
627    }
628    fUDM = NULL;
629}
630
631
632void SpoofData::removeReference() {
633    if (umtx_atomic_dec(&fRefCount) == 0) {
634        delete this;
635    }
636}
637
638
639SpoofData *SpoofData::addReference() {
640    umtx_atomic_inc(&fRefCount);
641    return this;
642}
643
644
645void *SpoofData::reserveSpace(int32_t numBytes,  UErrorCode &status) {
646    if (U_FAILURE(status)) {
647        return NULL;
648    }
649    if (!fDataOwned) {
650        U_ASSERT(FALSE);
651        status = U_INTERNAL_PROGRAM_ERROR;
652        return NULL;
653    }
654
655    numBytes = (numBytes + 15) & ~15;   // Round up to a multiple of 16
656    uint32_t returnOffset = fMemLimit;
657    fMemLimit += numBytes;
658    fRawData = static_cast<SpoofDataHeader *>(uprv_realloc(fRawData, fMemLimit));
659    fRawData->fLength = fMemLimit;
660    uprv_memset((char *)fRawData + returnOffset, 0, numBytes);
661    initPtrs(status);
662    return (char *)fRawData + returnOffset;
663}
664
665
666//----------------------------------------------------------------------------
667//
668//  ScriptSet implementation
669//
670//----------------------------------------------------------------------------
671ScriptSet::ScriptSet() {
672    for (uint32_t i=0; i<sizeof(bits)/sizeof(uint32_t); i++) {
673        bits[i] = 0;
674    }
675}
676
677ScriptSet::~ScriptSet() {
678}
679
680UBool ScriptSet::operator == (const ScriptSet &other) {
681    for (uint32_t i=0; i<sizeof(bits)/sizeof(uint32_t); i++) {
682        if (bits[i] != other.bits[i]) {
683            return FALSE;
684        }
685    }
686    return TRUE;
687}
688
689void ScriptSet::Union(UScriptCode script) {
690    uint32_t index = script / 32;
691    uint32_t bit   = 1 << (script & 31);
692    U_ASSERT(index < sizeof(bits)*4);
693    bits[index] |= bit;
694}
695
696
697void ScriptSet::Union(const ScriptSet &other) {
698    for (uint32_t i=0; i<sizeof(bits)/sizeof(uint32_t); i++) {
699        bits[i] |= other.bits[i];
700    }
701}
702
703void ScriptSet::intersect(const ScriptSet &other) {
704    for (uint32_t i=0; i<sizeof(bits)/sizeof(uint32_t); i++) {
705        bits[i] &= other.bits[i];
706    }
707}
708
709void ScriptSet::intersect(UScriptCode script) {
710    uint32_t index = script / 32;
711    uint32_t bit   = 1 << (script & 31);
712    U_ASSERT(index < sizeof(bits)*4);
713    uint32_t i;
714    for (i=0; i<index; i++) {
715        bits[i] = 0;
716    }
717    bits[index] &= bit;
718    for (i=index+1; i<sizeof(bits)/sizeof(uint32_t); i++) {
719        bits[i] = 0;
720    }
721}
722
723
724ScriptSet & ScriptSet::operator =(const ScriptSet &other) {
725    for (uint32_t i=0; i<sizeof(bits)/sizeof(uint32_t); i++) {
726        bits[i] = other.bits[i];
727    }
728    return *this;
729}
730
731
732void ScriptSet::setAll() {
733    for (uint32_t i=0; i<sizeof(bits)/sizeof(uint32_t); i++) {
734        bits[i] = 0xffffffffu;
735    }
736}
737
738
739void ScriptSet::resetAll() {
740    for (uint32_t i=0; i<sizeof(bits)/sizeof(uint32_t); i++) {
741        bits[i] = 0;
742    }
743}
744
745int32_t ScriptSet::countMembers() {
746    // This bit counter is good for sparse numbers of '1's, which is
747    //  very much the case that we will usually have.
748    int32_t count = 0;
749    for (uint32_t i=0; i<sizeof(bits)/sizeof(uint32_t); i++) {
750        uint32_t x = bits[i];
751        while (x > 0) {
752            count++;
753            x &= (x - 1);    // and off the least significant one bit.
754        }
755    }
756    return count;
757}
758
759
760
761//-----------------------------------------------------------------------------
762//
763//  NFKDBuffer Implementation.
764//
765//-----------------------------------------------------------------------------
766
767NFKDBuffer::NFKDBuffer(const UChar *text, int32_t length, UErrorCode &status) {
768    fNormalizedText = NULL;
769    fNormalizedTextLength = 0;
770    fOriginalText = text;
771    if (U_FAILURE(status)) {
772        return;
773    }
774    fNormalizedText = fSmallBuf;
775    fNormalizedTextLength = unorm_normalize(
776        text, length, UNORM_NFKD, 0, fNormalizedText, USPOOF_STACK_BUFFER_SIZE, &status);
777    if (status == U_BUFFER_OVERFLOW_ERROR) {
778        status = U_ZERO_ERROR;
779        fNormalizedText = (UChar *)uprv_malloc((fNormalizedTextLength+1)*sizeof(UChar));
780        if (fNormalizedText == NULL) {
781            status = U_MEMORY_ALLOCATION_ERROR;
782        } else {
783            fNormalizedTextLength = unorm_normalize(text, length, UNORM_NFKD, 0,
784                                        fNormalizedText, fNormalizedTextLength+1, &status);
785        }
786    }
787}
788
789
790NFKDBuffer::~NFKDBuffer() {
791    if (fNormalizedText != fSmallBuf) {
792        uprv_free(fNormalizedText);
793    }
794    fNormalizedText = 0;
795}
796
797const UChar *NFKDBuffer::getBuffer() {
798    return fNormalizedText;
799}
800
801int32_t NFKDBuffer::getLength() {
802    return fNormalizedTextLength;
803}
804
805
806
807
808
809U_NAMESPACE_END
810
811U_NAMESPACE_USE
812
813//-----------------------------------------------------------------------------
814//
815//  uspoof_swap   -  byte swap and char encoding swap of spoof data
816//
817//-----------------------------------------------------------------------------
818U_CAPI int32_t U_EXPORT2
819uspoof_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData,
820           UErrorCode *status) {
821
822    if (status == NULL || U_FAILURE(*status)) {
823        return 0;
824    }
825    if(ds==NULL || inData==NULL || length<-1 || (length>0 && outData==NULL)) {
826        *status=U_ILLEGAL_ARGUMENT_ERROR;
827        return 0;
828    }
829
830    //
831    //  Check that the data header is for spoof data.
832    //    (Header contents are defined in gencfu.cpp)
833    //
834    const UDataInfo *pInfo = (const UDataInfo *)((const char *)inData+4);
835    if(!(  pInfo->dataFormat[0]==0x43 &&   /* dataFormat="Cfu " */
836           pInfo->dataFormat[1]==0x66 &&
837           pInfo->dataFormat[2]==0x75 &&
838           pInfo->dataFormat[3]==0x20 &&
839           pInfo->formatVersion[0]==1  )) {
840        udata_printError(ds, "uspoof_swap(): data format %02x.%02x.%02x.%02x "
841                             "(format version %02x %02x %02x %02x) is not recognized\n",
842                         pInfo->dataFormat[0], pInfo->dataFormat[1],
843                         pInfo->dataFormat[2], pInfo->dataFormat[3],
844                         pInfo->formatVersion[0], pInfo->formatVersion[1],
845                         pInfo->formatVersion[2], pInfo->formatVersion[3]);
846        *status=U_UNSUPPORTED_ERROR;
847        return 0;
848    }
849
850    //
851    // Swap the data header.  (This is the generic ICU Data Header, not the uspoof Specific
852    //                         header).  This swap also conveniently gets us
853    //                         the size of the ICU d.h., which lets us locate the start
854    //                         of the uspoof specific data.
855    //
856    int32_t headerSize=udata_swapDataHeader(ds, inData, length, outData, status);
857
858
859    //
860    // Get the Spoof Data Header, and check that it appears to be OK.
861    //
862    //
863    const uint8_t   *inBytes =(const uint8_t *)inData+headerSize;
864    SpoofDataHeader *spoofDH = (SpoofDataHeader *)inBytes;
865    if (ds->readUInt32(spoofDH->fMagic)   != USPOOF_MAGIC ||
866        ds->readUInt32(spoofDH->fLength)  <  sizeof(SpoofDataHeader))
867    {
868        udata_printError(ds, "uspoof_swap(): Spoof Data header is invalid.\n");
869        *status=U_UNSUPPORTED_ERROR;
870        return 0;
871    }
872
873    //
874    // Prefight operation?  Just return the size
875    //
876    int32_t spoofDataLength = ds->readUInt32(spoofDH->fLength);
877    int32_t totalSize = headerSize + spoofDataLength;
878    if (length < 0) {
879        return totalSize;
880    }
881
882    //
883    // Check that length passed in is consistent with length from Spoof data header.
884    //
885    if (length < totalSize) {
886        udata_printError(ds, "uspoof_swap(): too few bytes (%d after ICU Data header) for spoof data.\n",
887                            spoofDataLength);
888        *status=U_INDEX_OUTOFBOUNDS_ERROR;
889        return 0;
890        }
891
892
893    //
894    // Swap the Data.  Do the data itself first, then the Spoof Data Header, because
895    //                 we need to reference the header to locate the data, and an
896    //                 inplace swap of the header leaves it unusable.
897    //
898    uint8_t          *outBytes = (uint8_t *)outData + headerSize;
899    SpoofDataHeader  *outputDH = (SpoofDataHeader *)outBytes;
900
901    int32_t   sectionStart;
902    int32_t   sectionLength;
903
904    //
905    // If not swapping in place, zero out the output buffer before starting.
906    //    Gaps may exist between the individual sections, and these must be zeroed in
907    //    the output buffer.  The simplest way to do that is to just zero the whole thing.
908    //
909    if (inBytes != outBytes) {
910        uprv_memset(outBytes, 0, spoofDataLength);
911    }
912
913    // Confusables Keys Section   (fCFUKeys)
914    sectionStart  = ds->readUInt32(spoofDH->fCFUKeys);
915    sectionLength = ds->readUInt32(spoofDH->fCFUKeysSize) * 4;
916    ds->swapArray32(ds, inBytes+sectionStart, sectionLength, outBytes+sectionStart, status);
917
918    // String Index Section
919    sectionStart  = ds->readUInt32(spoofDH->fCFUStringIndex);
920    sectionLength = ds->readUInt32(spoofDH->fCFUStringIndexSize) * 2;
921    ds->swapArray16(ds, inBytes+sectionStart, sectionLength, outBytes+sectionStart, status);
922
923    // String Table Section
924    sectionStart  = ds->readUInt32(spoofDH->fCFUStringTable);
925    sectionLength = ds->readUInt32(spoofDH->fCFUStringTableLen) * 2;
926    ds->swapArray16(ds, inBytes+sectionStart, sectionLength, outBytes+sectionStart, status);
927
928    // String Lengths Section
929    sectionStart  = ds->readUInt32(spoofDH->fCFUStringLengths);
930    sectionLength = ds->readUInt32(spoofDH->fCFUStringLengthsSize) * 4;
931    ds->swapArray16(ds, inBytes+sectionStart, sectionLength, outBytes+sectionStart, status);
932
933    // Any Case Trie
934    sectionStart  = ds->readUInt32(spoofDH->fAnyCaseTrie);
935    sectionLength = ds->readUInt32(spoofDH->fAnyCaseTrieLength);
936    utrie2_swap(ds, inBytes+sectionStart, sectionLength, outBytes+sectionStart, status);
937
938    // Lower Case Trie
939    sectionStart  = ds->readUInt32(spoofDH->fLowerCaseTrie);
940    sectionLength = ds->readUInt32(spoofDH->fLowerCaseTrieLength);
941    utrie2_swap(ds, inBytes+sectionStart, sectionLength, outBytes+sectionStart, status);
942
943    // Script Sets.  The data is an array of int32_t
944    sectionStart  = ds->readUInt32(spoofDH->fScriptSets);
945    sectionLength = ds->readUInt32(spoofDH->fScriptSetsLength) * 4;
946    ds->swapArray32(ds, inBytes+sectionStart, sectionLength, outBytes+sectionStart, status);
947
948    // And, last, swap the header itself.
949    //   int32_t   fMagic             // swap this
950    //   uint8_t   fFormatVersion[4]  // Do not swap this
951    //   int32_t   all the rest       // Swap the rest, all is 32 bit stuff.
952    //
953    uint32_t magic = ds->readUInt32(spoofDH->fMagic);
954    ds->writeUInt32((uint32_t *)&outputDH->fMagic, magic);
955    ds->swapArray32(ds, &spoofDH->fLength, sizeof(SpoofDataHeader)-8, &outputDH->fLength, status);
956
957    return totalSize;
958}
959
960#endif
961
962
963