1// Copyright (C) 2016 and later: Unicode, Inc. and others. 2// License & terms of use: http://www.unicode.org/copyright.html 3/* 4*************************************************************************** 5* Copyright (C) 2008-2013, International Business Machines Corporation 6* and others. All Rights Reserved. 7*************************************************************************** 8* 9* uspoof_impl.h 10* 11* Implemenation header for spoof detection 12* 13*/ 14 15#ifndef USPOOFIM_H 16#define USPOOFIM_H 17 18#include "uassert.h" 19#include "unicode/utypes.h" 20#include "unicode/uspoof.h" 21#include "unicode/uscript.h" 22#include "unicode/udata.h" 23 24#include "utrie2.h" 25 26#if !UCONFIG_NO_NORMALIZATION 27 28#ifdef __cplusplus 29 30U_NAMESPACE_BEGIN 31 32// The maximium length (in UTF-16 UChars) of the skeleton replacement string resulting from 33// a single input code point. This is function of the unicode.org data. 34#define USPOOF_MAX_SKELETON_EXPANSION 20 35 36// The default stack buffer size for copies or conversions or normalizations 37// of input strings being checked. (Used in multiple places.) 38#define USPOOF_STACK_BUFFER_SIZE 100 39 40// Magic number for sanity checking spoof data. 41#define USPOOF_MAGIC 0x3845fdef 42 43// Magic number for sanity checking spoof checkers. 44#define USPOOF_CHECK_MAGIC 0x2734ecde 45 46class ScriptSet; 47class SpoofData; 48struct SpoofDataHeader; 49class ConfusableDataUtils; 50 51/** 52 * Class SpoofImpl corresponds directly to the plain C API opaque type 53 * USpoofChecker. One can be cast to the other. 54 */ 55class SpoofImpl : public UObject { 56public: 57 SpoofImpl(SpoofData *data, UErrorCode& status); 58 SpoofImpl(UErrorCode& status); 59 SpoofImpl(); 60 void construct(UErrorCode& status); 61 virtual ~SpoofImpl(); 62 63 /** Copy constructor, used by the user level uspoof_clone() function. 64 */ 65 SpoofImpl(const SpoofImpl &src, UErrorCode &status); 66 67 USpoofChecker *asUSpoofChecker(); 68 static SpoofImpl *validateThis(USpoofChecker *sc, UErrorCode &status); 69 static const SpoofImpl *validateThis(const USpoofChecker *sc, UErrorCode &status); 70 71 /** Set and Get AllowedLocales, implementations of the corresponding API */ 72 void setAllowedLocales(const char *localesList, UErrorCode &status); 73 const char * getAllowedLocales(UErrorCode &status); 74 75 // Add (union) to the UnicodeSet all of the characters for the scripts used for 76 // the specified locale. Part of the implementation of setAllowedLocales. 77 void addScriptChars(const char *locale, UnicodeSet *allowedChars, UErrorCode &status); 78 79 // Functions implementing the features of UTS 39 section 5. 80 static void getAugmentedScriptSet(UChar32 codePoint, ScriptSet& result, UErrorCode& status); 81 void getResolvedScriptSet(const UnicodeString& input, ScriptSet& result, UErrorCode& status) const; 82 void getResolvedScriptSetWithout(const UnicodeString& input, UScriptCode script, ScriptSet& result, UErrorCode& status) const; 83 void getNumerics(const UnicodeString& input, UnicodeSet& result, UErrorCode& status) const; 84 URestrictionLevel getRestrictionLevel(const UnicodeString& input, UErrorCode& status) const; 85 86 /** parse a hex number. Untility used by the builders. */ 87 static UChar32 ScanHex(const UChar *s, int32_t start, int32_t limit, UErrorCode &status); 88 89 static UClassID U_EXPORT2 getStaticClassID(void); 90 virtual UClassID getDynamicClassID(void) const; 91 92 // 93 // Data Members 94 // 95 96 int32_t fMagic; // Internal sanity check. 97 int32_t fChecks; // Bit vector of checks to perform. 98 99 SpoofData *fSpoofData; 100 101 const UnicodeSet *fAllowedCharsSet; // The UnicodeSet of allowed characters. 102 // for this Spoof Checker. Defaults to all chars. 103 104 const char *fAllowedLocales; // The list of allowed locales. 105 URestrictionLevel fRestrictionLevel; // The maximum restriction level for an acceptable identifier. 106}; 107 108/** 109 * Class CheckResult corresponds directly to the plain C API opaque type 110 * USpoofCheckResult. One can be cast to the other. 111 */ 112class CheckResult : public UObject { 113public: 114 CheckResult(); 115 virtual ~CheckResult(); 116 117 USpoofCheckResult *asUSpoofCheckResult(); 118 static CheckResult *validateThis(USpoofCheckResult *ptr, UErrorCode &status); 119 static const CheckResult *validateThis(const USpoofCheckResult *ptr, UErrorCode &status); 120 121 void clear(); 122 123 // Used to convert this CheckResult to the older int32_t return value API 124 int32_t toCombinedBitmask(int32_t expectedChecks); 125 126 // Data Members 127 int32_t fMagic; // Internal sanity check. 128 int32_t fChecks; // Bit vector of checks that were failed. 129 UnicodeSet fNumerics; // Set of numerics found in the string. 130 URestrictionLevel fRestrictionLevel; // The restriction level of the string. 131}; 132 133 134// 135// Confusable Mappings Data Structures, version 2.0 136// 137// For the confusable data, we are essentially implementing a map, 138// key: a code point 139// value: a string. Most commonly one char in length, but can be more. 140// 141// The keys are stored as a sorted array of 32 bit ints. 142// bits 0-23 a code point value 143// bits 24-31 length of value string, in UChars (between 1 and 256 UChars). 144// The key table is sorted in ascending code point order. (not on the 145// 32 bit int value, the flag bits do not participate in the sorting.) 146// 147// Lookup is done by means of a binary search in the key table. 148// 149// The corresponding values are kept in a parallel array of 16 bit ints. 150// If the value string is of length 1, it is literally in the value array. 151// For longer strings, the value array contains an index into the strings table. 152// 153// String Table: 154// The strings table contains all of the value strings (those of length two or greater) 155// concatentated together into one long UChar (UTF-16) array. 156// 157// There is no nul character or other mark between adjacent strings. 158// 159//---------------------------------------------------------------------------- 160// 161// Changes from format version 1 to format version 2: 162// 1) Removal of the whole-script confusable data tables. 163// 2) Removal of the SL/SA/ML/MA and multi-table flags in the key bitmask. 164// 3) Expansion of string length value in the key bitmask from 2 bits to 8 bits. 165// 4) Removal of the string lengths table since 8 bits is sufficient for the 166// lengths of all entries in confusables.txt. 167 168 169 170// Internal functions for manipulating confusable data table keys 171#define USPOOF_CONFUSABLE_DATA_FORMAT_VERSION 2 // version for ICU 58 172class ConfusableDataUtils { 173public: 174 inline static UChar32 keyToCodePoint(int32_t key) { 175 return key & 0x00ffffff; 176 } 177 inline static int32_t keyToLength(int32_t key) { 178 return ((key & 0xff000000) >> 24) + 1; 179 } 180 inline static int32_t codePointAndLengthToKey(UChar32 codePoint, int32_t length) { 181 U_ASSERT((codePoint & 0x00ffffff) == codePoint); 182 U_ASSERT(length <= 256); 183 return codePoint | ((length - 1) << 24); 184 } 185}; 186 187 188//------------------------------------------------------------------------------------- 189// 190// SpoofData 191// 192// A small class that wraps the raw (usually memory mapped) spoof data. 193// Serves two primary functions: 194// 1. Convenience. Contains real pointers to the data, to avoid dealing with 195// the offsets in the raw data. 196// 2. Reference counting. When a spoof checker is cloned, the raw data is shared 197// and must be retained until all checkers using the data are closed. 198// Nothing in this struct includes state that is specific to any particular 199// USpoofDetector object. 200// 201//--------------------------------------------------------------------------------------- 202class SpoofData: public UMemory { 203 public: 204 static SpoofData* getDefault(UErrorCode &status); // Get standard ICU spoof data. 205 static void releaseDefault(); // Cleanup reference to default spoof data. 206 207 SpoofData(UErrorCode &status); // Create new spoof data wrapper. 208 // Only used when building new data from rules. 209 210 // Constructor for use when creating from prebuilt default data. 211 // A UDataMemory is what the ICU internal data loading functions provide. 212 // The udm is adopted by the SpoofData. 213 SpoofData(UDataMemory *udm, UErrorCode &status); 214 215 // Constructor for use when creating from serialized data. 216 // 217 SpoofData(const void *serializedData, int32_t length, UErrorCode &status); 218 219 // Check raw Spoof Data Version compatibility. 220 // Return TRUE it looks good. 221 UBool validateDataVersion(UErrorCode &status) const; 222 223 ~SpoofData(); // Destructor not normally used. 224 // Use removeReference() instead. 225 // Reference Counting functions. 226 // Clone of a user-level spoof detector increments the ref count on the data. 227 // Close of a user-level spoof detector decrements the ref count. 228 // If the data is owned by us, it will be deleted when count goes to zero. 229 SpoofData *addReference(); 230 void removeReference(); 231 232 // Reset all fields to an initial state. 233 // Called from the top of all constructors. 234 void reset(); 235 236 // Copy this instance's raw data buffer to the specified address. 237 int32_t serialize(void *buf, int32_t capacity, UErrorCode &status) const; 238 239 // Get the total number of bytes of data backed by this SpoofData. 240 // Not to be confused with length, which returns the number of confusable entries. 241 int32_t size() const; 242 243 // Get the confusable skeleton transform for a single code point. 244 // The result is a string with a length between 1 and 18 as of Unicode 9. 245 // This is the main public endpoint for this class. 246 // @return The length in UTF-16 code units of the substition string. 247 int32_t confusableLookup(UChar32 inChar, UnicodeString &dest) const; 248 249 // Get the number of confusable entries in this SpoofData. 250 int32_t length() const; 251 252 // Get the code point (key) at the specified index. 253 UChar32 codePointAt(int32_t index) const; 254 255 // Get the confusable skeleton (value) at the specified index. 256 // Append it to the specified UnicodeString&. 257 // @return The length in UTF-16 code units of the skeleton string. 258 int32_t appendValueTo(int32_t index, UnicodeString& dest) const; 259 260 private: 261 // Reserve space in the raw data. For use by builder when putting together a 262 // new set of data. Init the new storage to zero, to prevent inconsistent 263 // results if it is not all otherwise set by the requester. 264 // Return: 265 // pointer to the new space that was added by this function. 266 void *reserveSpace(int32_t numBytes, UErrorCode &status); 267 268 // initialize the pointers from this object to the raw data. 269 void initPtrs(UErrorCode &status); 270 271 SpoofDataHeader *fRawData; // Ptr to the raw memory-mapped data 272 UBool fDataOwned; // True if the raw data is owned, and needs 273 // to be deleted when refcount goes to zero. 274 UDataMemory *fUDM; // If not NULL, our data came from a 275 // UDataMemory, which we must close when 276 // we are done. 277 278 uint32_t fMemLimit; // Limit of available raw data space 279 u_atomic_int32_t fRefCount; 280 281 // Confusable data 282 int32_t *fCFUKeys; 283 uint16_t *fCFUValues; 284 UChar *fCFUStrings; 285 286 friend class ConfusabledataBuilder; 287}; 288 289//--------------------------------------------------------------------------------------- 290// 291// Raw Binary Data Formats, as loaded from the ICU data file, 292// or as built by the builder. 293// 294//--------------------------------------------------------------------------------------- 295struct SpoofDataHeader { 296 int32_t fMagic; // (0x3845fdef) 297 uint8_t fFormatVersion[4]; // Data Format. Same as the value in struct UDataInfo 298 // if there is one associated with this data. 299 int32_t fLength; // Total lenght in bytes of this spoof data, 300 // including all sections, not just the header. 301 302 // The following four sections refer to data representing the confusable data 303 // from the Unicode.org data from "confusables.txt" 304 305 int32_t fCFUKeys; // byte offset to Keys table (from SpoofDataHeader *) 306 int32_t fCFUKeysSize; // number of entries in keys table (32 bits each) 307 308 // TODO: change name to fCFUValues, for consistency. 309 int32_t fCFUStringIndex; // byte offset to String Indexes table 310 int32_t fCFUStringIndexSize; // number of entries in String Indexes table (16 bits each) 311 // (number of entries must be same as in Keys table 312 313 int32_t fCFUStringTable; // byte offset of String table 314 int32_t fCFUStringTableLen; // length of string table (in 16 bit UChars) 315 316 // The following sections are for data from xidmodifications.txt 317 318 int32_t unused[15]; // Padding, Room for Expansion 319}; 320 321 322 323U_NAMESPACE_END 324#endif /* __cplusplus */ 325 326/** 327 * Endianness swap function for binary spoof data. 328 * @internal 329 */ 330U_CAPI int32_t U_EXPORT2 331uspoof_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData, 332 UErrorCode *status); 333 334 335#endif 336 337#endif /* USPOOFIM_H */ 338 339