1/*
2***************************************************************************
3* Copyright (C) 2008-2013, International Business Machines Corporation
4* and others. All Rights Reserved.
5***************************************************************************
6*
7*  uspoof_impl.h
8*
9*    Implemenation header for spoof detection
10*
11*/
12
13#ifndef USPOOFIM_H
14#define USPOOFIM_H
15
16#include "unicode/utypes.h"
17#include "unicode/uspoof.h"
18#include "unicode/uscript.h"
19#include "unicode/udata.h"
20
21#include "utrie2.h"
22
23#if !UCONFIG_NO_NORMALIZATION
24
25#ifdef __cplusplus
26
27U_NAMESPACE_BEGIN
28
29// The maximium length (in UTF-16 UChars) of the skeleton replacement string resulting from
30//   a single input code point.  This is function of the unicode.org data.
31#define USPOOF_MAX_SKELETON_EXPANSION 20
32
33// The default stack buffer size for copies or conversions or normalizations
34// of input strings being checked.  (Used in multiple places.)
35#define USPOOF_STACK_BUFFER_SIZE 100
36
37// Magic number for sanity checking spoof data.
38#define USPOOF_MAGIC 0x3845fdef
39
40class IdentifierInfo;
41class ScriptSet;
42class SpoofData;
43struct SpoofDataHeader;
44struct SpoofStringLengthsElement;
45
46/**
47  *  Class SpoofImpl corresponds directly to the plain C API opaque type
48  *  USpoofChecker.  One can be cast to the other.
49  */
50class SpoofImpl : public UObject  {
51public:
52	SpoofImpl(SpoofData *data, UErrorCode &status);
53	SpoofImpl();
54	virtual ~SpoofImpl();
55
56    /** Copy constructor, used by the user level uspoof_clone() function.
57     */
58    SpoofImpl(const SpoofImpl &src, UErrorCode &status);
59
60    static SpoofImpl *validateThis(USpoofChecker *sc, UErrorCode &status);
61    static const SpoofImpl *validateThis(const USpoofChecker *sc, UErrorCode &status);
62
63    /** Get the confusable skeleton transform for a single code point.
64     *  The result is a string with a length between 1 and 18.
65     *  @param    tableMask  bit flag specifying which confusable table to use.
66     *                       One of USPOOF_SL_TABLE_FLAG, USPOOF_MA_TABLE_FLAG, etc.
67     *  @return   The length in UTF-16 code units of the substition string.
68     */
69    int32_t confusableLookup(UChar32 inChar, int32_t tableMask, UnicodeString &destBuf) const;
70
71    /** Set and Get AllowedLocales, implementations of the corresponding API */
72    void setAllowedLocales(const char *localesList, UErrorCode &status);
73    const char * getAllowedLocales(UErrorCode &status);
74
75    // Add (union) to the UnicodeSet all of the characters for the scripts used for
76    // the specified locale.  Part of the implementation of setAllowedLocales.
77    void addScriptChars(const char *locale, UnicodeSet *allowedChars, UErrorCode &status);
78
79
80    /** parse a hex number.  Untility used by the builders.   */
81    static UChar32 ScanHex(const UChar *s, int32_t start, int32_t limit, UErrorCode &status);
82
83    // Implementation for Whole Script tests.
84    // Return the test bit flag to be ORed into the eventual user return value
85    //    if a Spoof opportunity is detected.
86    void wholeScriptCheck(
87        const UnicodeString &text, ScriptSet *result, UErrorCode &status) const;
88
89    static UClassID U_EXPORT2 getStaticClassID(void);
90    virtual UClassID getDynamicClassID(void) const;
91
92    // IdentifierInfo Cache. IdentifierInfo objects are somewhat expensive to create.
93    //                       Maintain a one-element cache, which is sufficient to avoid repeatedly
94    //                       creating new ones unless we get multi-thread concurrency in spoof
95    //                       check operations, which should be statistically uncommon.
96    IdentifierInfo *getIdentifierInfo(UErrorCode &status) const;
97    void releaseIdentifierInfo(IdentifierInfo *idInfo) const;
98
99    //
100    // Data Members
101    //
102
103    int32_t           fMagic;             // Internal sanity check.
104    int32_t           fChecks;            // Bit vector of checks to perform.
105
106    SpoofData        *fSpoofData;
107
108    const UnicodeSet *fAllowedCharsSet;   // The UnicodeSet of allowed characters.
109                                          //   for this Spoof Checker.  Defaults to all chars.
110
111    const char       *fAllowedLocales;    // The list of allowed locales.
112    URestrictionLevel fRestrictionLevel;  // The maximum restriction level for an acceptable identifier.
113
114    IdentifierInfo    *fCachedIdentifierInfo;    // Do not use directly. See getIdentifierInfo().:w
115};
116
117
118
119//
120//  Confusable Mappings Data Structures
121//
122//    For the confusable data, we are essentially implementing a map,
123//       key:    a code point
124//       value:  a string.  Most commonly one char in length, but can be more.
125//
126//    The keys are stored as a sorted array of 32 bit ints.
127//             bits 0-23    a code point value
128//             bits 24-31   flags
129//                24:  1 if entry applies to SL table
130//                25:  1 if entry applies to SA table
131//                26:  1 if entry applies to ML table
132//                27:  1 if entry applies to MA table
133//                28:  1 if there are multiple entries for this code point.
134//                29-30:  length of value string, in UChars.
135//                         values are (1, 2, 3, other)
136//        The key table is sorted in ascending code point order.  (not on the
137//        32 bit int value, the flag bits do not participate in the sorting.)
138//
139//        Lookup is done by means of a binary search in the key table.
140//
141//    The corresponding values are kept in a parallel array of 16 bit ints.
142//        If the value string is of length 1, it is literally in the value array.
143//        For longer strings, the value array contains an index into the strings table.
144//
145//    String Table:
146//       The strings table contains all of the value strings (those of length two or greater)
147//       concatentated together into one long UChar (UTF-16) array.
148//
149//       The array is arranged by length of the strings - all strings of the same length
150//       are stored together.  The sections are ordered by length of the strings -
151//       all two char strings first, followed by all of the three Char strings, etc.
152//
153//       There is no nul character or other mark between adjacent strings.
154//
155//    String Lengths table
156//       The length of strings from 1 to 3 is flagged in the key table.
157//       For strings of length 4 or longer, the string length table provides a
158//       mapping between an index into the string table and the corresponding length.
159//       Strings of these lengths are rare, so lookup time is not an issue.
160//       Each entry consists of
161//            uint16_t      index of the _last_ string with this length
162//            uint16_t      the length
163//
164
165// Flag bits in the Key entries
166#define USPOOF_SL_TABLE_FLAG (1<<24)
167#define USPOOF_SA_TABLE_FLAG (1<<25)
168#define USPOOF_ML_TABLE_FLAG (1<<26)
169#define USPOOF_MA_TABLE_FLAG (1<<27)
170#define USPOOF_KEY_MULTIPLE_VALUES (1<<28)
171#define USPOOF_KEY_LENGTH_SHIFT 29
172#define USPOOF_KEY_LENGTH_FIELD(x) (((x)>>29) & 3)
173
174
175struct SpoofStringLengthsElement {
176    uint16_t      fLastString;         // index in string table of last string with this length
177    uint16_t      fStrLength;           // Length of strings
178};
179
180
181
182//-------------------------------------------------------------------------------------
183//
184//  SpoofData
185//
186//    A small class that wraps the raw (usually memory mapped) spoof data.
187//    Serves two primary functions:
188//      1.  Convenience.  Contains real pointers to the data, to avoid dealing with
189//          the offsets in the raw data.
190//      2.  Reference counting.  When a spoof checker is cloned, the raw data is shared
191//          and must be retained until all checkers using the data are closed.
192//    Nothing in this struct includes state that is specific to any particular
193//    USpoofDetector object.
194//
195//---------------------------------------------------------------------------------------
196class SpoofData: public UMemory {
197  public:
198    static SpoofData *getDefault(UErrorCode &status);   // Load standard ICU spoof data.
199    SpoofData(UErrorCode &status);   // Create new spoof data wrapper.
200                                     // Only used when building new data from rules.
201
202    // Constructor for use when creating from prebuilt default data.
203    //   A UDataMemory is what the ICU internal data loading functions provide.
204    //   The udm is adopted by the SpoofData.
205    SpoofData(UDataMemory *udm, UErrorCode &status);
206
207    // Constructor for use when creating from serialized data.
208    //
209    SpoofData(const void *serializedData, int32_t length, UErrorCode &status);
210
211    //  Check raw Spoof Data Version compatibility.
212    //  Return TRUE it looks good.
213    static UBool validateDataVersion(const SpoofDataHeader *rawData, UErrorCode &status);
214    ~SpoofData();                    // Destructor not normally used.
215                                     // Use removeReference() instead.
216    // Reference Counting functions.
217    //    Clone of a user-level spoof detector increments the ref count on the data.
218    //    Close of a user-level spoof detector decrements the ref count.
219    //    If the data is owned by us, it will be deleted when count goes to zero.
220    SpoofData *addReference();
221    void removeReference();
222
223    // Reserve space in the raw data.  For use by builder when putting together a
224    //   new set of data.  Init the new storage to zero, to prevent inconsistent
225    //   results if it is not all otherwise set by the requester.
226    //  Return:
227    //    pointer to the new space that was added by this function.
228    void *reserveSpace(int32_t numBytes, UErrorCode &status);
229
230    // initialize the pointers from this object to the raw data.
231    void initPtrs(UErrorCode &status);
232
233    // Reset all fields to an initial state.
234    // Called from the top of all constructors.
235    void reset();
236
237    SpoofDataHeader             *fRawData;          // Ptr to the raw memory-mapped data
238    UBool                       fDataOwned;         // True if the raw data is owned, and needs
239                                                    //  to be deleted when refcount goes to zero.
240    UDataMemory                 *fUDM;              // If not NULL, our data came from a
241                                                    //   UDataMemory, which we must close when
242                                                    //   we are done.
243
244    uint32_t                    fMemLimit;          // Limit of available raw data space
245    u_atomic_int32_t            fRefCount;
246
247    // Confusable data
248    int32_t                     *fCFUKeys;
249    uint16_t                    *fCFUValues;
250    SpoofStringLengthsElement   *fCFUStringLengths;
251    UChar                       *fCFUStrings;
252
253    // Whole Script Confusable Data
254    UTrie2                      *fAnyCaseTrie;
255    UTrie2                      *fLowerCaseTrie;
256    ScriptSet                   *fScriptSets;
257    };
258
259
260//---------------------------------------------------------------------------------------
261//
262//  Raw Binary Data Formats, as loaded from the ICU data file,
263//    or as built by the builder.
264//
265//---------------------------------------------------------------------------------------
266struct SpoofDataHeader {
267    int32_t       fMagic;                // (0x3845fdef)
268    uint8_t       fFormatVersion[4];     // Data Format. Same as the value in struct UDataInfo
269                                         //   if there is one associated with this data.
270    int32_t       fLength;               // Total lenght in bytes of this spoof data,
271                                         //   including all sections, not just the header.
272
273    // The following four sections refer to data representing the confusable data
274    //   from the Unicode.org data from "confusables.txt"
275
276    int32_t       fCFUKeys;               // byte offset to Keys table (from SpoofDataHeader *)
277    int32_t       fCFUKeysSize;           // number of entries in keys table  (32 bits each)
278
279    // TODO: change name to fCFUValues, for consistency.
280    int32_t       fCFUStringIndex;        // byte offset to String Indexes table
281    int32_t       fCFUStringIndexSize;    // number of entries in String Indexes table (16 bits each)
282                                          //     (number of entries must be same as in Keys table
283
284    int32_t       fCFUStringTable;        // byte offset of String table
285    int32_t       fCFUStringTableLen;     // length of string table (in 16 bit UChars)
286
287    int32_t       fCFUStringLengths;      // byte offset to String Lengths table
288    int32_t       fCFUStringLengthsSize;  // number of entries in lengths table. (2 x 16 bits each)
289
290
291    // The following sections are for data from confusablesWholeScript.txt
292
293    int32_t       fAnyCaseTrie;           // byte offset to the serialized Any Case Trie
294    int32_t       fAnyCaseTrieLength;     // Length (bytes) of the serialized Any Case Trie
295
296    int32_t       fLowerCaseTrie;         // byte offset to the serialized Lower Case Trie
297    int32_t       fLowerCaseTrieLength;   // Length (bytes) of the serialized Lower Case Trie
298
299    int32_t       fScriptSets;            // byte offset to array of ScriptSets
300    int32_t       fScriptSetsLength;      // Number of ScriptSets (24 bytes each)
301
302
303    // The following sections are for data from xidmodifications.txt
304
305
306    int32_t       unused[15];              // Padding, Room for Expansion
307
308 };
309
310
311
312
313//
314//  Structure for the Whole Script Confusable Data
315//    See Unicode UAX-39, Unicode Security Mechanisms, for a description of the
316//    Whole Script confusable data
317//
318//  The data provides mappings from code points to a set of scripts
319//    that contain characters that might be confused with the code point.
320//  There are two mappings, one for lower case only, and one for characters
321//    of any case.
322//
323//  The actual data consists of a utrie2 to map from a code point to an offset,
324//  and an array of UScriptSets (essentially bit maps) that is indexed
325//  by the offsets obtained from the Trie.
326//
327//
328
329
330U_NAMESPACE_END
331#endif /* __cplusplus */
332
333/**
334  * Endianness swap function for binary spoof data.
335  * @internal
336  */
337U_CAPI int32_t U_EXPORT2
338uspoof_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData,
339            UErrorCode *status);
340
341
342#endif
343
344#endif  /* USPOOFIM_H */
345
346