1/* 2******************************************************************************* 3* 4* Copyright (C) 1998-2011, International Business Machines 5* Corporation and others. All Rights Reserved. 6* 7******************************************************************************* 8* 9* Private implementation header for C collation 10* file name: ucol_imp.h 11* encoding: US-ASCII 12* tab size: 8 (not used) 13* indentation:4 14* 15* created on: 2000dec11 16* created by: Vladimir Weinstein 17* 18* Modification history 19* Date Name Comments 20* 02/16/2001 synwee Added UCOL_GETPREVCE for the use in ucoleitr 21* 02/27/2001 synwee Added getMaxExpansion data structure in UCollator 22* 03/02/2001 synwee Added UCOL_IMPLICIT_CE 23* 03/12/2001 synwee Added pointer start to collIterate. 24*/ 25 26#ifndef UCOL_IMP_H 27#define UCOL_IMP_H 28 29#include "unicode/utypes.h" 30 31#define UCA_DATA_TYPE "icu" 32#define UCA_DATA_NAME "ucadata" 33#define INVC_DATA_TYPE "icu" 34#define INVC_DATA_NAME "invuca" 35 36/** 37 * Convenience string denoting the Collation data tree 38 * @internal ICU 3.0 39 */ 40#define U_ICUDATA_COLL U_ICUDATA_NAME U_TREE_SEPARATOR_STRING "coll" 41 42#if !UCONFIG_NO_COLLATION 43 44#ifdef XP_CPLUSPLUS 45#include "unicode/normalizer2.h" 46#include "unicode/unistr.h" 47#endif 48#include "unicode/ucol.h" 49#include "utrie.h" 50#include "cmemory.h" 51 52/* This is the internal header file which contains important declarations for 53 * the collation framework. 54 * Ready to use collators are stored as binary images. Both UCA and tailorings 55 * share the same binary format. Individual files (currently only UCA) have a 56 * udata header in front of the image and should be opened using udata_open. 57 * Tailoring images are currently stored inside resource bundles and are intialized 58 * through ucol_open API. 59 * 60 * The following describes the formats for collation binaries 61 * (UCA & tailorings) and for the inverse UCA table. 62 * Substructures are described in the collation design document at 63 * http://source.icu-project.org/repos/icu/icuhtml/trunk/design/collation/ICU_collation_design.htm 64 * 65 * ------------------------------------------------------------- 66 * 67 * Here is the format of binary collation image. 68 * 69 * Physical order of structures: 70 * - header (UCATableHeader) 71 * - options (UColOptionSet) 72 * - expansions (CE[]) 73 * - contractions (UChar[contractionSize] + CE[contractionSize]) 74 * - serialized UTrie with mappings of code points to CEs 75 * - max expansion tables (CE[endExpansionCECount] + uint8_t[endExpansionCECount]) 76 * - two bit sets for backward processing in strcoll (identical prefixes) 77 * and for backward CE iteration (each set is uint8_t[UCOL_UNSAFECP_TABLE_SIZE]) 78 * - UCA constants (UCAConstants) 79 * - UCA contractions (UChar[contractionUCACombosSize][contractionUCACombosWidth]) 80 * 81 * UCATableHeader fields: 82 * 83 * int32_t size; - image size in bytes 84 * 85 * Offsets to interesting data. All offsets are in bytes. 86 * to get the address add to the header address and cast properly. 87 * Some offsets are zero if the corresponding structures are empty. 88 * 89 * Tailoring binaries that only set options and contain no mappings etc. 90 * will have all offsets 0 except for the options and expansion offsets, 91 * which give the position and length of the options array. 92 * 93 * uint32_t options; - offset to default collator options (UColOptionSet *), 94 * a set of 32-bit values. See declaration of UColOptionSet for more details 95 * 96 * uint32_t UCAConsts; - only used (!=0) in UCA image - structure which holds values for indirect positioning and implicit ranges 97 * See declaration of UCAConstants structure. This is a set of unsigned 32-bit values used to store 98 * important constant values that are defined in the UCA and used for building and runtime. 99 * 100 * uint32_t contractionUCACombos; - only used (!=0) in UCA image - list of UCA contractions. This is a zero terminated array of UChar[contractionUCACombosWidth], 101 * containing contractions from the UCA. These are needed in the build process to copy UCA contractions 102 * in case the base contraction symbol is tailored. 103 * 104 * uint32_t magic; - must contain UCOL_HEADER_MAGIC (formatVersion 2.3) 105 * 106 * uint32_t mappingPosition; - offset to UTrie (const uint8_t *mappingPosition). This is a serialized UTrie and should be treated as such. 107 * Used as a primary lookup table for collation elements. 108 * 109 * uint32_t expansion; - offset to expansion table (uint32_t *expansion). This is an array of expansion CEs. Never 0. 110 * 111 * uint32_t contractionIndex; - offset to contraction table (UChar *contractionIndex). Used to look up contraction sequences. Contents 112 * are aligned with the contents of contractionCEs table. 0 if no contractions. 113 * 114 * uint32_t contractionCEs; - offset to resulting contraction CEs (uint32_t *contractionCEs). When a contraction is resolved in the 115 * in the contractionIndex table, the resulting index is used to look up corresponding CE in this table. 116 * 0 if no contractions. 117 * uint32_t contractionSize; - size of contraction table in elements (both Index and CEs). 118 * 119 * Tables described below are used for Boyer-Moore searching algorithm - they define the size of longest expansion 120 * and last CEs in expansions. 121 * uint32_t endExpansionCE; - offset to array of last collation element in expansion (uint32_t *). 122 * Never 0. 123 * uint32_t expansionCESize; - array of maximum expansion sizes (uint8_t *) 124 * int32_t endExpansionCECount; - size of endExpansionCE. See UCOL_GETMAXEXPANSION 125 * for the usage model 126 * 127 * These two offsets point to byte tables that are used in the backup heuristics. 128 * uint32_t unsafeCP; - hash table of unsafe code points (uint8_t *). See ucol_unsafeCP function. 129 * uint32_t contrEndCP; - hash table of final code points in contractions (uint8_t *). See ucol_contractionEndCP. 130 * 131 * int32_t contractionUCACombosSize; - number of UChar[contractionUCACombosWidth] in contractionUCACombos 132 * (formatVersion 2.3) 133 * UBool jamoSpecial; - Jamo special indicator (uint8_t). If TRUE, Jamos are special, so we cannot use simple Hangul decomposition. 134 * UBool isBigEndian; - endianness of this collation binary (formatVersion 2.3) 135 * uint8_t charSetFamily; - charset family of this collation binary (formatVersion 2.3) 136 * uint8_t contractionUCACombosWidth; - number of UChars per UCA contraction in contractionUCACombos (formatVersion 2.3) 137 * 138 * Various version fields 139 * UVersionInfo version; - version 4 uint8_t 140 * UVersionInfo UCAVersion; - version 4 uint8_t 141 * UVersionInfo UCDVersion; - version 4 uint8_t 142 * UVersionInfo formatVersion; - version of the format of the collation binary 143 * same formatVersion as in ucadata.icu's UDataInfo header 144 * (formatVersion 2.3) 145 * 146 * uint32_t offset to the reordering code to lead CE byte remapping table 147 * uint32_t offset to the lead CE byte to reordering code mapping table 148 * 149 * uint8_t reserved[76]; - currently unused 150 * 151 * ------------------------------------------------------------- 152 * 153 * Inverse UCA is used for constructing collators from rules. It is always an individual file 154 * and always has a UDataInfo header. 155 * here is the structure: 156 * 157 * uint32_t byteSize; - size of inverse UCA image in bytes 158 * uint32_t tableSize; - length of inverse table (number of uint32_t[3] rows) 159 * uint32_t contsSize; - size of continuation table (number of UChars in table) 160 * 161 * uint32_t table; - offset to inverse table (uint32_t *) 162 * Inverse table contains of rows of 3 uint32_t values. First two values are CE and a possible continuation 163 * the third value is either a code unit (if there is only one code unit for element) or an index to continuation 164 * (number of code units combined with an index). 165 * table. If more than one codepoint have the same CE, continuation table contains code units separated by FFFF and final 166 * code unit sequence for a CE is terminated by FFFE. 167 * uint32_t conts; - offset to continuation table (uint16_t *). Contains code units that transform to a same CE. 168 * 169 * UVersionInfo UCAVersion; - version of the UCA, read from file 4 uint8_t 170 * uint8_t padding[8]; - padding 8 uint8_t 171 * Header is followed by the table and continuation table. 172*/ 173 174/* let us know whether reserved fields are reset to zero or junked */ 175#define UCOL_HEADER_MAGIC 0x20030618 176 177/* UDataInfo for UCA mapping table */ 178/* dataFormat="UCol" */ 179#define UCA_DATA_FORMAT_0 ((uint8_t)0x55) 180#define UCA_DATA_FORMAT_1 ((uint8_t)0x43) 181#define UCA_DATA_FORMAT_2 ((uint8_t)0x6f) 182#define UCA_DATA_FORMAT_3 ((uint8_t)0x6c) 183 184#define UCA_FORMAT_VERSION_0 ((uint8_t)3) 185#define UCA_FORMAT_VERSION_1 0 186#define UCA_FORMAT_VERSION_2 ((uint8_t)0) 187#define UCA_FORMAT_VERSION_3 ((uint8_t)0) 188 189/* UDataInfo for inverse UCA table */ 190/* dataFormat="InvC" */ 191#define INVUCA_DATA_FORMAT_0 ((uint8_t)0x49) 192#define INVUCA_DATA_FORMAT_1 ((uint8_t)0x6E) 193#define INVUCA_DATA_FORMAT_2 ((uint8_t)0x76) 194#define INVUCA_DATA_FORMAT_3 ((uint8_t)0x43) 195 196#define INVUCA_FORMAT_VERSION_0 ((uint8_t)2) 197#define INVUCA_FORMAT_VERSION_1 ((uint8_t)1) 198#define INVUCA_FORMAT_VERSION_2 ((uint8_t)0) 199#define INVUCA_FORMAT_VERSION_3 ((uint8_t)0) 200 201/* This is the size of the stack allocated buffer for sortkey generation and similar operations */ 202/* if it is too small, heap allocation will occur.*/ 203/* you can change this value if you need memory - it will affect the performance, though, since we're going to malloc */ 204#define UCOL_MAX_BUFFER 128 205#define UCOL_PRIMARY_MAX_BUFFER 8*UCOL_MAX_BUFFER 206#define UCOL_SECONDARY_MAX_BUFFER UCOL_MAX_BUFFER 207#define UCOL_TERTIARY_MAX_BUFFER UCOL_MAX_BUFFER 208/* 209#define UCOL_CASE_MAX_BUFFER UCOL_MAX_BUFFER/4 210 211UCOL_CASE_MAX_BUFFER as previously defined above was too small. A single collation element can 212generate two caseShift values, and UCOL_CASE_SHIFT_START (=7) caseShift values are compressed into 213one byte. UCOL_MAX_BUFFER should effectively be multipled by 2/UCOL_CASE_SHIFT_START (2/7), not 1/4. 214Perhaps UCOL_CASE_SHIFT_START used to be 8; then this would have been correct. We should dynamically 215define UCOL_CASE_MAX_BUFFER in terms of both UCOL_MAX_BUFFER and UCOL_CASE_SHIFT_START. Since 216UCOL_CASE_SHIFT_START is defined lower down, we move the real definition of UCOL_CASE_MAX_BUFFER 217after it, further down. 218*/ 219#define UCOL_QUAD_MAX_BUFFER 2*UCOL_MAX_BUFFER 220 221#define UCOL_NORMALIZATION_GROWTH 2 222#define UCOL_NORMALIZATION_MAX_BUFFER UCOL_MAX_BUFFER*UCOL_NORMALIZATION_GROWTH 223 224/* This writable buffer is used if we encounter Thai and need to reorder the string on the fly */ 225/* Sometimes we already have a writable buffer (like in case of normalized strings). */ 226/* 227you can change this value to any value >= 4 if you need memory - 228it will affect the performance, though, since we're going to malloc. 229Note 3 is the minimum value for Thai collation and 4 is the 230minimum number for special Jamo 231*/ 232#define UCOL_WRITABLE_BUFFER_SIZE 256 233 234/* This is the size of the buffer for expansion CE's */ 235/* In reality we should not have to deal with expm sequences longer then 16 */ 236/* you can change this value if you need memory */ 237/* WARNING THIS BUFFER DOES HAVE MALLOC FALLBACK. If you make it too small, you'll get into performance trouble */ 238/* Reasonable small value is around 10, if you don't do Arabic or other funky collations that have long expansion sequence */ 239/* This is the longest expansion sequence we can handle without bombing out */ 240#define UCOL_EXPAND_CE_BUFFER_SIZE 64 241 242/* This is the size to increase the buffer for expansion CE's */ 243#define UCOL_EXPAND_CE_BUFFER_EXTEND_SIZE 64 244 245 246/* Unsafe UChar hash table table size. */ 247/* size is 32 bytes for 1 bit for each latin 1 char + some power of two for */ 248/* hashing the rest of the chars. Size in bytes */ 249#define UCOL_UNSAFECP_TABLE_SIZE 1056 250 /* mask value down to "some power of two"-1 */ 251 /* number of bits, not num of bytes. */ 252#define UCOL_UNSAFECP_TABLE_MASK 0x1fff 253 254 255/* flags bits for collIterate.flags */ 256/* */ 257/* NORM - set for incremental normalize of source string */ 258#define UCOL_ITER_NORM 1 259 260#define UCOL_ITER_HASLEN 2 261 262 /* UCOL_ITER_INNORMBUF - set if the "pos" is in */ 263 /* the writable side buffer, handling */ 264 /* incrementally normalized characters. */ 265#define UCOL_ITER_INNORMBUF 4 266 267 /* UCOL_ITER_ALLOCATED - set if this iterator has */ 268 /* malloced storage to expand a buffer. */ 269#define UCOL_ITER_ALLOCATED 8 270 /* UCOL_HIRAGANA_Q - note if the codepoint was hiragana */ 271#define UCOL_HIRAGANA_Q 16 272 /* UCOL_WAS_HIRAGANA - set to TRUE if there was a Hiragana */ 273 /* otherwise set to false */ 274#define UCOL_WAS_HIRAGANA 32 275 /* UCOL_USE_ITERATOR - set this if collIterate uses a */ 276 /* character iterator instead of simply accessing string */ 277 /* by index */ 278#define UCOL_USE_ITERATOR 64 279 280#define UCOL_FORCE_HAN_IMPLICIT 128 281 282#define NFC_ZERO_CC_BLOCK_LIMIT_ 0x300 283 284#ifdef XP_CPLUSPLUS 285 286U_NAMESPACE_BEGIN 287 288typedef struct collIterate : public UMemory { 289 const UChar *string; /* Original string */ 290 /* UChar *start; Pointer to the start of the source string. Either points to string 291 or to writableBuffer */ 292 const UChar *endp; /* string end ptr. Is undefined for null terminated strings */ 293 const UChar *pos; /* This is position in the string. Can be to original or writable buf */ 294 295 uint32_t *toReturn; /* This is the CE from CEs buffer that should be returned */ 296 uint32_t *CEpos; /* This is the position to which we have stored processed CEs */ 297 298 int32_t *offsetReturn; /* This is the offset to return, if non-NULL */ 299 int32_t *offsetStore; /* This is the pointer for storing offsets */ 300 int32_t offsetRepeatCount; /* Repeat stored offset if non-zero */ 301 int32_t offsetRepeatValue; /* offset value to repeat */ 302 303 UnicodeString writableBuffer; 304 const UChar *fcdPosition; /* Position in the original string to continue FCD check from. */ 305 const UCollator *coll; 306 const Normalizer2 *nfd; 307 uint8_t flags; 308 uint8_t origFlags; 309 uint32_t *extendCEs; /* This is use if CEs is not big enough */ 310 int32_t extendCEsSize; /* Holds the size of the dynamic CEs buffer */ 311 uint32_t CEs[UCOL_EXPAND_CE_BUFFER_SIZE]; /* This is where we store CEs */ 312 313 int32_t *offsetBuffer; /* A dynamic buffer to hold offsets */ 314 int32_t offsetBufferSize; /* The size of the offset buffer */ 315 316 UCharIterator *iterator; 317 /*int32_t iteratorIndex;*/ 318 319 // The offsetBuffer should probably be a UVector32, but helper functions 320 // are an improvement over duplicated code. 321 void appendOffset(int32_t offset, UErrorCode &errorCode); 322} collIterate; 323 324U_NAMESPACE_END 325 326#else 327 328typedef struct collIterate collIterate; 329 330#endif 331 332#define paddedsize(something) ((something)+((((something)%4)!=0)?(4-(something)%4):0)) 333#define headersize (paddedsize(sizeof(UCATableHeader))+paddedsize(sizeof(UColOptionSet))) 334 335/* 336struct used internally in getSpecial*CE. 337data similar to collIterate. 338*/ 339struct collIterateState { 340 const UChar *pos; /* This is position in the string. Can be to original or writable buf */ 341 const UChar *returnPos; 342 const UChar *fcdPosition; /* Position in the original string to continue FCD check from. */ 343 const UChar *bufferaddress; /* address of the normalization buffer */ 344 int32_t buffersize; 345 uint8_t flags; 346 uint8_t origFlags; 347 uint32_t iteratorIndex; 348 int32_t iteratorMove; 349}; 350 351U_CAPI void U_EXPORT2 352uprv_init_collIterate(const UCollator *collator, 353 const UChar *sourceString, int32_t sourceLen, 354 U_NAMESPACE_QUALIFIER collIterate *s, UErrorCode *status); 355 356/* Internal functions for C test code. */ 357U_CAPI U_NAMESPACE_QUALIFIER collIterate * U_EXPORT2 358uprv_new_collIterate(UErrorCode *status); 359 360U_CAPI void U_EXPORT2 361uprv_delete_collIterate(U_NAMESPACE_QUALIFIER collIterate *s); 362 363/* @return s->pos == s->endp */ 364U_CAPI UBool U_EXPORT2 365uprv_collIterateAtEnd(U_NAMESPACE_QUALIFIER collIterate *s); 366 367#ifdef XP_CPLUSPLUS 368 369U_NAMESPACE_BEGIN 370 371struct UCollationPCE; 372typedef struct UCollationPCE UCollationPCE; 373 374U_NAMESPACE_END 375 376struct UCollationElements : public U_NAMESPACE_QUALIFIER UMemory 377{ 378 /** 379 * Struct wrapper for source data 380 */ 381 U_NAMESPACE_QUALIFIER collIterate iteratordata_; 382 /** 383 * Indicates if this data has been reset. 384 */ 385 UBool reset_; 386 /** 387 * Indicates if the data should be deleted. 388 */ 389 UBool isWritable; 390 391/** 392 * Data for getNextProcessed, getPreviousProcessed. 393 */ 394 U_NAMESPACE_QUALIFIER UCollationPCE *pce; 395}; 396 397#else 398/*opaque type*/ 399struct UCollationElements; 400#endif 401 402U_CAPI void U_EXPORT2 403uprv_init_pce(const struct UCollationElements *elems); 404 405#define UCOL_LEVELTERMINATOR 1 406 407/* mask off anything but primary order */ 408#define UCOL_PRIMARYORDERMASK 0xffff0000 409/* mask off anything but secondary order */ 410#define UCOL_SECONDARYORDERMASK 0x0000ff00 411/* mask off anything but tertiary order */ 412#define UCOL_TERTIARYORDERMASK 0x000000ff 413/* primary order shift */ 414#define UCOL_PRIMARYORDERSHIFT 16 415/* secondary order shift */ 416#define UCOL_SECONDARYORDERSHIFT 8 417 418#define UCOL_BYTE_SIZE_MASK 0xFF 419 420#define UCOL_CASE_BYTE_START 0x80 421#define UCOL_CASE_SHIFT_START 7 422 423/* 424The definition of UCOL_CASE_MAX_BUFFER is moved down here so it can use UCOL_CASE_SHIFT_START. 425 426A single collation element can generate two caseShift values, and UCOL_CASE_SHIFT_START caseShift 427values are compressed into one byte. The UCOL_CASE_MAX_BUFFER should effectively be UCOL_MAX_BUFFER 428multipled by 2/UCOL_CASE_SHIFT_START, with suitable rounding up. 429*/ 430#define UCOL_CASE_MAX_BUFFER (((2*UCOL_MAX_BUFFER) + UCOL_CASE_SHIFT_START - 1)/UCOL_CASE_SHIFT_START) 431 432#define UCOL_IGNORABLE 0 433 434/* get weights from a CE */ 435#define UCOL_PRIMARYORDER(order) (((order) & UCOL_PRIMARYORDERMASK)>> UCOL_PRIMARYORDERSHIFT) 436#define UCOL_SECONDARYORDER(order) (((order) & UCOL_SECONDARYORDERMASK)>> UCOL_SECONDARYORDERSHIFT) 437#define UCOL_TERTIARYORDER(order) ((order) & UCOL_TERTIARYORDERMASK) 438 439/** 440 * Determine if a character is a Thai vowel (which sorts after 441 * its base consonant). 442 */ 443#define UCOL_ISTHAIPREVOWEL(ch) ((((uint32_t)(ch) - 0xe40) <= (0xe44 - 0xe40)) || \ 444 (((uint32_t)(ch) - 0xec0) <= (0xec4 - 0xec0))) 445 446/** 447 * Determine if a character is a Thai base consonant 448 */ 449#define UCOL_ISTHAIBASECONSONANT(ch) ((uint32_t)(ch) - 0xe01) <= (0xe2e - 0xe01) 450 451#define UCOL_ISJAMO(ch) ((((uint32_t)(ch) - 0x1100) <= (0x1112 - 0x1100)) || \ 452 (((uint32_t)(ch) - 0x1161) <= (0x1175 - 0x1161)) || \ 453 (((uint32_t)(ch) - 0x11A8) <= (0x11C2 - 0x11A8))) 454 455/* Han character ranges */ 456#define UCOL_FIRST_HAN 0x4E00 457#define UCOL_LAST_HAN 0x9FFF 458#define UCOL_FIRST_HAN_A 0x3400 459#define UCOL_LAST_HAN_A 0x4DBF 460#define UCOL_FIRST_HAN_COMPAT 0xFAE0 461#define UCOL_LAST_HAN_COMPAT 0xFA2F 462 463/* Han extension B is in plane 2 */ 464#define UCOL_FIRST_HAN_B 0x20000 465#define UCOL_LAST_HAN_B 0x2A6DF 466 467/* Hangul range */ 468#define UCOL_FIRST_HANGUL 0xAC00 469#define UCOL_LAST_HANGUL 0xD7AF 470 471/* Jamo ranges */ 472#define UCOL_FIRST_L_JAMO 0x1100 473#define UCOL_FIRST_V_JAMO 0x1161 474#define UCOL_FIRST_T_JAMO 0x11A8 475#define UCOL_LAST_T_JAMO 0x11F9 476 477 478#if 0 479/* initializes collIterate structure */ 480/* made as macro to speed up things */ 481#define init_collIterate(collator, sourceString, sourceLen, s) { \ 482 (s)->start = (s)->string = (s)->pos = (UChar *)(sourceString); \ 483 (s)->endp = (sourceLen) == -1 ? NULL :(UChar *)(sourceString)+(sourceLen); \ 484 (s)->CEpos = (s)->toReturn = (s)->CEs; \ 485 (s)->isThai = TRUE; \ 486 (s)->writableBuffer = (s)->stackWritableBuffer; \ 487 (s)->writableBufSize = UCOL_WRITABLE_BUFFER_SIZE; \ 488 (s)->coll = (collator); \ 489 (s)->fcdPosition = 0; \ 490 (s)->flags = 0; \ 491 if(((collator)->normalizationMode == UCOL_ON)) (s)->flags |= UCOL_ITER_NORM; \ 492} 493#endif 494 495 496 497/* 498* Macro to get the maximum size of an expansion ending with the argument ce. 499* Used in the Boyer Moore algorithm. 500* Note for tailoring, the UCA maxexpansion table has been merged. 501* Hence we only have to search the tailored collator only. 502* @param coll const UCollator pointer 503* @param order last collation element of the expansion sequence 504* @param result size of the longest expansion with argument collation element 505* as the last element 506*/ 507#define UCOL_GETMAXEXPANSION(coll, order, result) { \ 508 const uint32_t *start; \ 509 const uint32_t *limit; \ 510 const uint32_t *mid; \ 511 start = (coll)->endExpansionCE; \ 512 limit = (coll)->lastEndExpansionCE; \ 513 while (start < limit - 1) { \ 514 mid = start + ((limit - start) >> 1); \ 515 if ((order) <= *mid) { \ 516 limit = mid; \ 517 } \ 518 else { \ 519 start = mid; \ 520 } \ 521 } \ 522 if (*start == order) { \ 523 result = *((coll)->expansionCESize + (start - (coll)->endExpansionCE)); \ 524 } \ 525 else if (*limit == order) { \ 526 result = *(coll->expansionCESize + (limit - coll->endExpansionCE)); \ 527 } \ 528 else if ((order & 0xFFFF) == 0x00C0) { \ 529 result = 2; \ 530 } \ 531 else { \ 532 result = 1; \ 533 } \ 534} 535 536U_CFUNC 537uint32_t ucol_prv_getSpecialCE(const UCollator *coll, UChar ch, uint32_t CE, 538 U_NAMESPACE_QUALIFIER collIterate *source, UErrorCode *status); 539 540U_CFUNC 541uint32_t ucol_prv_getSpecialPrevCE(const UCollator *coll, UChar ch, uint32_t CE, 542 U_NAMESPACE_QUALIFIER collIterate *source, UErrorCode *status); 543U_CAPI uint32_t U_EXPORT2 ucol_getNextCE(const UCollator *coll, 544 U_NAMESPACE_QUALIFIER collIterate *collationSource, UErrorCode *status); 545U_CFUNC uint32_t U_EXPORT2 ucol_getPrevCE(const UCollator *coll, 546 U_NAMESPACE_QUALIFIER collIterate *collationSource, 547 UErrorCode *status); 548/* get some memory */ 549void *ucol_getABuffer(const UCollator *coll, uint32_t size); 550 551#ifdef XP_CPLUSPLUS 552 553U_NAMESPACE_BEGIN 554 555class SortKeyByteSink; 556 557U_NAMESPACE_END 558 559/* function used by C++ getCollationKey to prevent restarting the calculation */ 560U_CFUNC int32_t 561ucol_getSortKeyWithAllocation(const UCollator *coll, 562 const UChar *source, int32_t sourceLength, 563 uint8_t *&result, int32_t &resultCapacity, 564 UErrorCode *pErrorCode); 565 566typedef void U_CALLCONV 567SortKeyGenerator(const UCollator *coll, 568 const UChar *source, 569 int32_t sourceLength, 570 U_NAMESPACE_QUALIFIER SortKeyByteSink &result, 571 UErrorCode *status); 572 573/* worker function for generating sortkeys */ 574U_CFUNC 575void U_CALLCONV 576ucol_calcSortKey(const UCollator *coll, 577 const UChar *source, 578 int32_t sourceLength, 579 U_NAMESPACE_QUALIFIER SortKeyByteSink &result, 580 UErrorCode *status); 581 582U_CFUNC 583void U_CALLCONV 584ucol_calcSortKeySimpleTertiary(const UCollator *coll, 585 const UChar *source, 586 int32_t sourceLength, 587 U_NAMESPACE_QUALIFIER SortKeyByteSink &result, 588 UErrorCode *status); 589 590#else 591 592typedef void U_CALLCONV 593SortKeyGenerator(const UCollator *coll, 594 const UChar *source, 595 int32_t sourceLength, 596 void *result, 597 UErrorCode *status); 598 599#endif 600 601/** 602 * Makes a copy of the Collator's rule data. The format is 603 * that of .col files. 604 * 605 * @param length returns the length of the data, in bytes. 606 * @param status the error status 607 * @return memory, owned by the caller, of size 'length' bytes. 608 * @internal INTERNAL USE ONLY 609 */ 610U_CFUNC uint8_t* U_EXPORT2 611ucol_cloneRuleData(const UCollator *coll, int32_t *length, UErrorCode *status); 612 613/** 614 * Used to set requested and valid locales on a collator returned by the collator 615 * service. 616 */ 617U_CFUNC void U_EXPORT2 618ucol_setReqValidLocales(UCollator *coll, char *requestedLocaleToAdopt, char *validLocaleToAdopt, char *actualLocaleToAdopt); 619 620#define UCOL_SPECIAL_FLAG 0xF0000000 621#define UCOL_TAG_SHIFT 24 622#define UCOL_TAG_MASK 0x0F000000 623#define INIT_EXP_TABLE_SIZE 1024 624#define UCOL_NOT_FOUND 0xF0000000 625#define UCOL_EXPANSION 0xF1000000 626#define UCOL_CONTRACTION 0xF2000000 627#define UCOL_THAI 0xF3000000 628#define UCOL_UNMARKED 0x03 629#define UCOL_NEW_TERTIARYORDERMASK 0x0000003f 630 631/* Bit mask for primary collation strength. */ 632#define UCOL_PRIMARYMASK 0xFFFF0000 633 634/* Bit mask for secondary collation strength. */ 635#define UCOL_SECONDARYMASK 0x0000FF00 636 637/* Bit mask for tertiary collation strength. */ 638#define UCOL_TERTIARYMASK 0x000000FF 639 640/** 641 * Internal. 642 * This indicates the last element in a UCollationElements has been consumed. 643 * Compare with the UCOL_NULLORDER, UCOL_NULLORDER is returned if error occurs. 644 */ 645#define UCOL_NO_MORE_CES 0x00010101 646#define UCOL_NO_MORE_CES_PRIMARY 0x00010000 647#define UCOL_NO_MORE_CES_SECONDARY 0x00000100 648#define UCOL_NO_MORE_CES_TERTIARY 0x00000001 649 650#define isSpecial(CE) ((((CE)&UCOL_SPECIAL_FLAG)>>28)==0xF) 651 652#define UCOL_UPPER_CASE 0x80 653#define UCOL_MIXED_CASE 0x40 654#define UCOL_LOWER_CASE 0x00 655 656#define UCOL_CONTINUATION_MARKER 0xC0 657#define UCOL_REMOVE_CONTINUATION 0xFFFFFF3F 658 659#define isContinuation(CE) (((CE) & UCOL_CONTINUATION_MARKER) == UCOL_CONTINUATION_MARKER) 660#define isFlagged(CE) (((CE) & 0x80) == 0x80) 661#define isLongPrimary(CE) (((CE) & 0xC0) == 0xC0) 662 663#define getCETag(CE) (((CE)&UCOL_TAG_MASK)>>UCOL_TAG_SHIFT) 664#define isContraction(CE) (isSpecial((CE)) && (getCETag((CE)) == CONTRACTION_TAG)) 665#define isPrefix(CE) (isSpecial((CE)) && (getCETag((CE)) == SPEC_PROC_TAG)) 666#define constructContractCE(tag, CE) (UCOL_SPECIAL_FLAG | ((tag)<<UCOL_TAG_SHIFT) | ((CE)&0xFFFFFF)) 667#define constructSpecProcCE(CE) (UCOL_SPECIAL_FLAG | (SPEC_PROC_TAG<<UCOL_TAG_SHIFT) | ((CE)&0xFFFFFF)) 668#define getContractOffset(CE) ((CE)&0xFFFFFF) 669#define getExpansionOffset(CE) (((CE)&0x00FFFFF0)>>4) 670#define getExpansionCount(CE) ((CE)&0xF) 671#define isCEIgnorable(CE) (((CE) & 0xFFFFFFBF) == 0) 672 673/* StringSearch internal use */ 674#define inNormBuf(coleiter) ((coleiter)->iteratordata_.flags & UCOL_ITER_INNORMBUF) 675#define isFCDPointerNull(coleiter) ((coleiter)->iteratordata_.fcdPosition == NULL) 676#define hasExpansion(coleiter) ((coleiter)->iteratordata_.CEpos != (coleiter)->iteratordata_.CEs) 677#define getExpansionPrefix(coleiter) ((coleiter)->iteratordata_.toReturn - (coleiter)->iteratordata_.CEs) 678#define setExpansionPrefix(coleiter, offset) ((coleiter)->iteratordata_.CEs + offset) 679#define getExpansionSuffix(coleiter) ((coleiter)->iteratordata_.CEpos - (coleiter)->iteratordata_.toReturn) 680#define setExpansionSuffix(coleiter, offset) ((coleiter)->iteratordata_.toReturn = (coleiter)->iteratordata_.CEpos - leftoverces) 681 682/* This is an enum that lists magic special byte values from the fractional UCA. 683 * See also http://site.icu-project.org/design/collation/bytes */ 684/* TODO: all the #defines that refer to special byte values from the UCA should be changed to point here */ 685 686enum { 687 UCOL_BYTE_ZERO = 0x00, 688 UCOL_BYTE_LEVEL_SEPARATOR = 0x01, 689 UCOL_BYTE_SORTKEY_GLUE = 0x02, 690 UCOL_BYTE_SHIFT_PREFIX = 0x03, 691 UCOL_BYTE_UNSHIFTED_MIN = UCOL_BYTE_SHIFT_PREFIX, 692 UCOL_BYTE_FIRST_TAILORED = 0x04, 693 UCOL_BYTE_COMMON = 0x05, 694 UCOL_BYTE_FIRST_UCA = UCOL_BYTE_COMMON, 695 /* TODO: Make the following values dynamic since they change with almost every UCA version. */ 696 UCOL_CODAN_PLACEHOLDER = 0x12, 697 UCOL_BYTE_FIRST_NON_LATIN_PRIMARY = 0x5B, 698 UCOL_BYTE_UNSHIFTED_MAX = 0xFF 699}; 700 701#if 0 702#define UCOL_RESET_TOP_VALUE 0x9F000303 703#define UCOL_FIRST_PRIMARY_IGNORABLE 0x00008705 704#define UCOL_LAST_PRIMARY_IGNORABLE 0x0000DD05 705#define UCOL_LAST_PRIMARY_IGNORABLE_CONT 0x000051C0 706#define UCOL_FIRST_SECONDARY_IGNORABLE 0x00000000 707#define UCOL_LAST_SECONDARY_IGNORABLE 0x00000500 708#define UCOL_FIRST_TERTIARY_IGNORABLE 0x00000000 709#define UCOL_LAST_TERTIARY_IGNORABLE 0x00000000 710#define UCOL_FIRST_VARIABLE 0x05070505 711#define UCOL_LAST_VARIABLE 0x179B0505 712#define UCOL_FIRST_NON_VARIABLE 0x1A200505 713#define UCOL_LAST_NON_VARIABLE 0x7B41058F 714 715#define UCOL_NEXT_TOP_VALUE 0xE8960303 716#define UCOL_NEXT_FIRST_PRIMARY_IGNORABLE 0x00008905 717#define UCOL_NEXT_LAST_PRIMARY_IGNORABLE 0x03000303 718#define UCOL_NEXT_FIRST_SECONDARY_IGNORABLE 0x00008705 719#define UCOL_NEXT_LAST_SECONDARY_IGNORABLE 0x00000500 720#define UCOL_NEXT_FIRST_TERTIARY_IGNORABLE 0x00000000 721#define UCOL_NEXT_LAST_TERTIARY_IGNORABLE 0x00000000 722#define UCOL_NEXT_FIRST_VARIABLE 0x05090505 723#define UCOL_NEXT_LAST_VARIABLE 0x1A200505 724 725#define PRIMARY_IMPLICIT_MIN 0xE8000000 726#define PRIMARY_IMPLICIT_MAX 0xF0000000 727#endif 728 729/* These constants can be changed - sortkey size is affected by them */ 730#define UCOL_PROPORTION2 0.5 731#define UCOL_PROPORTION3 0.667 732 733/* These values come from the UCA */ 734#define UCOL_COMMON_BOT2 UCOL_BYTE_COMMON 735#define UCOL_COMMON_TOP2 0x86u 736#define UCOL_TOTAL2 (UCOL_COMMON_TOP2-UCOL_COMMON_BOT2-1) 737 738#define UCOL_FLAG_BIT_MASK_CASE_SW_OFF 0x80 739#define UCOL_FLAG_BIT_MASK_CASE_SW_ON 0x40 740#define UCOL_COMMON_TOP3_CASE_SW_OFF 0x85 741#define UCOL_COMMON_TOP3_CASE_SW_LOWER 0x45 742#define UCOL_COMMON_TOP3_CASE_SW_UPPER 0xC5 743 744/* These values come from the UCA */ 745#define UCOL_COMMON_BOT3 0x05 746 747#define UCOL_COMMON_BOTTOM3_CASE_SW_UPPER 0x86; 748#define UCOL_COMMON_BOTTOM3_CASE_SW_LOWER UCOL_COMMON_BOT3; 749 750#define UCOL_TOP_COUNT2 (UCOL_PROPORTION2*UCOL_TOTAL2) 751#define UCOL_BOT_COUNT2 (UCOL_TOTAL2-UCOL_TOP_COUNT2) 752 753 754#define UCOL_COMMON2 UCOL_COMMON_BOT2 755#define UCOL_COMMON3_UPPERFIRST 0xC5 756#define UCOL_COMMON3_NORMAL UCOL_COMMON_BOT3 757 758#define UCOL_COMMON4 0xFF 759 760/* constants for case level/case first handling */ 761/* used to instantiate UCollators fields in ucol_updateInternalState */ 762#define UCOL_CASE_SWITCH 0xC0 763#define UCOL_NO_CASE_SWITCH 0x00 764 765#define UCOL_REMOVE_CASE 0x3F 766#define UCOL_KEEP_CASE 0xFF 767 768#define UCOL_CASE_BIT_MASK 0xC0 769 770#define UCOL_TERT_CASE_MASK 0xFF 771 772#define UCOL_ENDOFLATINONERANGE 0xFF 773#define UCOL_LATINONETABLELEN (UCOL_ENDOFLATINONERANGE+50) 774#define UCOL_BAIL_OUT_CE 0xFF000000 775 776 777typedef enum { 778 NOT_FOUND_TAG = 0, 779 EXPANSION_TAG = 1, /* This code point results in an expansion */ 780 CONTRACTION_TAG = 2, /* Start of a contraction */ 781 THAI_TAG = 3, /* Thai character - do the reordering */ 782 CHARSET_TAG = 4, /* Charset processing, not yet implemented */ 783 SURROGATE_TAG = 5, /* Lead surrogate that is tailored and doesn't start a contraction */ 784 HANGUL_SYLLABLE_TAG = 6, /* AC00-D7AF*/ 785 LEAD_SURROGATE_TAG = 7, /* D800-DBFF*/ 786 TRAIL_SURROGATE_TAG = 8, /* DC00-DFFF*/ 787 CJK_IMPLICIT_TAG = 9, /* 0x3400-0x4DB5, 0x4E00-0x9FA5, 0xF900-0xFA2D*/ 788 IMPLICIT_TAG = 10, 789 SPEC_PROC_TAG = 11, 790 /* ICU 2.1 */ 791 LONG_PRIMARY_TAG = 12, /* This is a three byte primary with starting secondaries and tertiaries */ 792 /* It fits in a single 32 bit CE and is used instead of expansion to save */ 793 /* space without affecting the performance (hopefully) */ 794 795 DIGIT_TAG = 13, /* COllate Digits As Numbers (CODAN) implementation */ 796 797 CE_TAGS_COUNT 798} UColCETags; 799 800/* 801 ***************************************************************************************** 802 * set to zero 803 * NON_CHARACTER FDD0 - FDEF, FFFE, FFFF, 1FFFE, 1FFFF, 2FFFE, 2FFFF,...e.g. **FFFE, **FFFF 804 ****************************************************************************************** 805 */ 806 807typedef struct { 808 uint32_t variableTopValue; 809 /*UColAttributeValue*/ int32_t frenchCollation; 810 /*UColAttributeValue*/ int32_t alternateHandling; /* attribute for handling variable elements*/ 811 /*UColAttributeValue*/ int32_t caseFirst; /* who goes first, lower case or uppercase */ 812 /*UColAttributeValue*/ int32_t caseLevel; /* do we have an extra case level */ 813 /*UColAttributeValue*/ int32_t normalizationMode; /* attribute for normalization */ 814 /*UColAttributeValue*/ int32_t strength; /* attribute for strength */ 815 /*UColAttributeValue*/ int32_t hiraganaQ; /* attribute for special Hiragana */ 816 /*UColAttributeValue*/ int32_t numericCollation; /* attribute for numeric collation */ 817 uint32_t reserved[15]; /* for future use */ 818} UColOptionSet; 819 820typedef struct { 821 uint32_t UCA_FIRST_TERTIARY_IGNORABLE[2]; /*0x00000000*/ 822 uint32_t UCA_LAST_TERTIARY_IGNORABLE[2]; /*0x00000000*/ 823 uint32_t UCA_FIRST_PRIMARY_IGNORABLE[2]; /*0x00008705*/ 824 uint32_t UCA_FIRST_SECONDARY_IGNORABLE[2]; /*0x00000000*/ 825 uint32_t UCA_LAST_SECONDARY_IGNORABLE[2]; /*0x00000500*/ 826 uint32_t UCA_LAST_PRIMARY_IGNORABLE[2]; /*0x0000DD05*/ 827 uint32_t UCA_FIRST_VARIABLE[2]; /*0x05070505*/ 828 uint32_t UCA_LAST_VARIABLE[2]; /*0x13CF0505*/ 829 uint32_t UCA_FIRST_NON_VARIABLE[2]; /*0x16200505*/ 830 uint32_t UCA_LAST_NON_VARIABLE[2]; /*0x767C0505*/ 831 uint32_t UCA_RESET_TOP_VALUE[2]; /*0x9F000303*/ 832 uint32_t UCA_FIRST_IMPLICIT[2]; 833 uint32_t UCA_LAST_IMPLICIT[2]; 834 uint32_t UCA_FIRST_TRAILING[2]; 835 uint32_t UCA_LAST_TRAILING[2]; 836 837#if 0 838 uint32_t UCA_NEXT_TOP_VALUE[2]; /*0xE8960303*/ 839 uint32_t UCA_NEXT_FIRST_PRIMARY_IGNORABLE; /*0x00008905*/ 840 uint32_t UCA_NEXT_LAST_PRIMARY_IGNORABLE; /*0x03000303*/ 841 uint32_t UCA_NEXT_FIRST_SECONDARY_IGNORABLE; /*0x00008705*/ 842 uint32_t UCA_NEXT_LAST_SECONDARY_IGNORABLE; /*0x00000500*/ 843 uint32_t UCA_NEXT_FIRST_TERTIARY_IGNORABLE; /*0x00000000*/ 844 uint32_t UCA_NEXT_LAST_TERTIARY_IGNORABLE; /*0x00000000*/ 845 uint32_t UCA_NEXT_FIRST_VARIABLE; /*0x05090505*/ 846 uint32_t UCA_NEXT_LAST_VARIABLE; /*0x16200505*/ 847#endif 848 849 uint32_t UCA_PRIMARY_TOP_MIN; 850 uint32_t UCA_PRIMARY_IMPLICIT_MIN; /*0xE8000000*/ 851 uint32_t UCA_PRIMARY_IMPLICIT_MAX; /*0xF0000000*/ 852 uint32_t UCA_PRIMARY_TRAILING_MIN; /*0xE8000000*/ 853 uint32_t UCA_PRIMARY_TRAILING_MAX; /*0xF0000000*/ 854 uint32_t UCA_PRIMARY_SPECIAL_MIN; /*0xE8000000*/ 855 uint32_t UCA_PRIMARY_SPECIAL_MAX; /*0xF0000000*/ 856} UCAConstants; 857 858typedef struct { 859 int32_t size; 860 /* all the offsets are in bytes */ 861 /* to get the address add to the header address and cast properly */ 862 uint32_t options; /* these are the default options for the collator */ 863 uint32_t UCAConsts; /* structure which holds values for indirect positioning and implicit ranges */ 864 uint32_t contractionUCACombos; /* this one is needed only for UCA, to copy the appropriate contractions */ 865 uint32_t magic; /* magic number - lets us know whether reserved data is reset or junked */ 866 uint32_t mappingPosition; /* const uint8_t *mappingPosition; */ 867 uint32_t expansion; /* uint32_t *expansion; */ 868 uint32_t contractionIndex; /* UChar *contractionIndex; */ 869 uint32_t contractionCEs; /* uint32_t *contractionCEs; */ 870 uint32_t contractionSize; /* needed for various closures */ 871 /*int32_t latinOneMapping;*/ /* this is now handled in the trie itself *//* fast track to latin1 chars */ 872 873 uint32_t endExpansionCE; /* array of last collation element in 874 expansion */ 875 uint32_t expansionCESize; /* array of maximum expansion size 876 corresponding to the expansion 877 collation elements with last element 878 in endExpansionCE*/ 879 int32_t endExpansionCECount; /* size of endExpansionCE */ 880 uint32_t unsafeCP; /* hash table of unsafe code points */ 881 uint32_t contrEndCP; /* hash table of final code points */ 882 /* in contractions. */ 883 884 int32_t contractionUCACombosSize; /* number of UCA contraction items. */ 885 /*Length is contractionUCACombosSize*contractionUCACombosWidth*sizeof(UChar) */ 886 UBool jamoSpecial; /* is jamoSpecial */ 887 UBool isBigEndian; /* is this data big endian? from the UDataInfo header*/ 888 uint8_t charSetFamily; /* what is the charset family of this data from the UDataInfo header*/ 889 uint8_t contractionUCACombosWidth; /* width of UCA combos field */ 890 UVersionInfo version; 891 UVersionInfo UCAVersion; /* version of the UCA, read from file */ 892 UVersionInfo UCDVersion; /* UCD version, obtained by u_getUnicodeVersion */ 893 UVersionInfo formatVersion; /* format version from the UDataInfo header */ 894 uint32_t scriptToLeadByte; /* offset to script to lead collation byte mapping data */ 895 uint32_t leadByteToScript; /* offset to lead collation byte to script mapping data */ 896 uint8_t reserved[76]; /* for future use */ 897} UCATableHeader; 898 899#define U_UNKNOWN_STATE 0 900#define U_COLLATOR_STATE 0x01 901#define U_STATE_LIMIT 0x02 902 903/* This is the first structure in a state */ 904/* it should be machine independent */ 905typedef struct { 906 /* this structure is supposed to be readable on all the platforms.*/ 907 /* first 2 fields hold the size of the structure in a platform independent way */ 908 uint8_t sizeLo; 909 uint8_t sizeHi; 910 /* identifying the writing platform */ 911 uint8_t isBigEndian; 912 /* see U_CHARSET_FAMILY values in utypes.h */ 913 uint8_t charsetFamily; 914 /* version of ICU this state structure comes from */ 915 uint8_t icuVersion[4]; 916 /* What is the data following this state */ 917 uint8_t type; 918 /* more stuff to come, keep it on 16 byte boundary */ 919 uint8_t reserved[7]; 920} UStateStruct; 921 922/* This structure follows UStatusStruct */ 923/* and contains data specific for the collators */ 924/* Endianess needs to be decided before accessing this structure */ 925/* However, it's size IS endianess independent */ 926typedef struct { 927 /* size of this structure */ 928 uint8_t sizeLo; 929 uint8_t sizeHi; 930 /* This state is followed by the frozen tailoring */ 931 uint8_t containsTailoring; 932 /* This state is followed by the frozen UCA */ 933 uint8_t containsUCA; 934 /* Version info - the same one */ 935 uint8_t versionInfo[4]; 936 937 /* for charset CEs */ 938 uint8_t charsetName[32]; 939 /* this is the resolved locale name*/ 940 uint8_t locale[32]; 941 942 /* Attributes. Open ended */ 943 /* all the following will be moved to uint32_t because of portability */ 944 /* variable top value */ 945 uint32_t variableTopValue; 946 /* attribute for handling variable elements*/ 947 uint32_t /*UColAttributeValue*/ alternateHandling; 948 /* how to handle secondary weights */ 949 uint32_t /*UColAttributeValue*/ frenchCollation; 950 /* who goes first, lower case or uppercase */ 951 uint32_t /*UColAttributeValue*/ caseFirst; 952 /* do we have an extra case level */ 953 uint32_t /*UColAttributeValue*/ caseLevel; 954 /* attribute for normalization */ 955 uint32_t /*UColAttributeValue*/ normalizationMode; 956 /* attribute for strength */ 957 uint32_t /*UColAttributeValue*/ strength; 958 /* to be immediately 16 byte aligned */ 959 uint8_t reserved[12]; 960} UColStateStruct; 961 962#define UCOL_INV_SIZEMASK 0xFFF00000 963#define UCOL_INV_OFFSETMASK 0x000FFFFF 964#define UCOL_INV_SHIFTVALUE 20 965 966U_CDECL_BEGIN 967 968typedef struct { 969 uint32_t byteSize; 970 uint32_t tableSize; 971 uint32_t contsSize; 972 uint32_t table; 973 uint32_t conts; 974 UVersionInfo UCAVersion; /* version of the UCA, read from file */ 975 uint8_t padding[8]; 976} InverseUCATableHeader; 977 978typedef void U_CALLCONV 979ResourceCleaner(UCollator *coll); 980 981 982struct UCollator { 983 UColOptionSet *options; 984 SortKeyGenerator *sortKeyGen; 985 uint32_t *latinOneCEs; 986 char* actualLocale; 987 char* validLocale; 988 char* requestedLocale; 989 const UChar *rules; 990 const UChar *ucaRules; 991 const UCollator *UCA; 992 const UCATableHeader *image; 993 UTrie mapping; 994 const uint32_t *latinOneMapping; 995 const uint32_t *expansion; 996 const UChar *contractionIndex; 997 const uint32_t *contractionCEs; 998 999 const uint32_t *endExpansionCE; /* array of last ces in an expansion ce. 1000 corresponds to expansionCESize */ 1001 const uint32_t *lastEndExpansionCE;/* pointer to the last element in endExpansionCE */ 1002 const uint8_t *expansionCESize; /* array of the maximum size of a 1003 expansion ce with the last ce 1004 corresponding to endExpansionCE, 1005 terminated with a null */ 1006 const uint8_t *unsafeCP; /* unsafe code points hashtable */ 1007 const uint8_t *contrEndCP; /* Contraction ending chars hash table */ 1008 UChar minUnsafeCP; /* Smallest unsafe Code Point. */ 1009 UChar minContrEndCP; /* Smallest code point at end of a contraction */ 1010 1011 int32_t rulesLength; 1012 int32_t latinOneTableLen; 1013 1014 uint32_t variableTopValue; 1015 UColAttributeValue frenchCollation; 1016 UColAttributeValue alternateHandling; /* attribute for handling variable elements*/ 1017 UColAttributeValue caseFirst; /* who goes first, lower case or uppercase */ 1018 UColAttributeValue caseLevel; /* do we have an extra case level */ 1019 UColAttributeValue normalizationMode; /* attribute for normalization */ 1020 UColAttributeValue strength; /* attribute for strength */ 1021 UColAttributeValue hiraganaQ; /* attribute for Hiragana */ 1022 UColAttributeValue numericCollation; 1023 UBool variableTopValueisDefault; 1024 UBool frenchCollationisDefault; 1025 UBool alternateHandlingisDefault; /* attribute for handling variable elements*/ 1026 UBool caseFirstisDefault; /* who goes first, lower case or uppercase */ 1027 UBool caseLevelisDefault; /* do we have an extra case level */ 1028 UBool normalizationModeisDefault; /* attribute for normalization */ 1029 UBool strengthisDefault; /* attribute for strength */ 1030 UBool hiraganaQisDefault; /* attribute for Hiragana */ 1031 UBool numericCollationisDefault; 1032 UBool hasRealData; /* some collators have only options, like French, no rules */ 1033 /* to speed up things, we use the UCA image, but we don't want it */ 1034 /* to run around */ 1035 1036 UBool freeOnClose; 1037 UBool freeOptionsOnClose; 1038 UBool freeRulesOnClose; 1039 UBool freeImageOnClose; 1040 UBool freeDefaultReorderCodesOnClose; 1041 UBool freeReorderCodesOnClose; 1042 UBool freeLeadBytePermutationTableOnClose; 1043 1044 UBool latinOneUse; 1045 UBool latinOneRegenTable; 1046 UBool latinOneFailed; 1047 1048 int8_t tertiaryAddition; /* when switching case, we need to add or subtract different values */ 1049 uint8_t caseSwitch; 1050 uint8_t tertiaryCommon; 1051 uint8_t tertiaryMask; 1052 uint8_t tertiaryTop; /* Upper range when compressing */ 1053 uint8_t tertiaryBottom; /* Upper range when compressing */ 1054 uint8_t tertiaryTopCount; 1055 uint8_t tertiaryBottomCount; 1056 1057 UVersionInfo dataVersion; /* Data info of UCA table */ 1058 int32_t* defaultReorderCodes; 1059 int32_t defaultReorderCodesLength; 1060 int32_t* reorderCodes; 1061 int32_t reorderCodesLength; 1062 uint8_t* leadBytePermutationTable; 1063}; 1064 1065U_CDECL_END 1066 1067/* various internal functions */ 1068 1069/* do not close UCA returned by ucol_initUCA! */ 1070U_CFUNC 1071UCollator* ucol_initUCA(UErrorCode *status); 1072 1073U_CFUNC 1074UCollator* ucol_initCollator(const UCATableHeader *image, UCollator *fillIn, const UCollator *UCA, UErrorCode *status); 1075 1076U_CFUNC 1077void ucol_setOptionsFromHeader(UCollator* result, UColOptionSet * opts, UErrorCode *status); 1078 1079U_CFUNC 1080UCollator* ucol_open_internal(const char* loc, UErrorCode* status); 1081 1082#if 0 1083U_CFUNC 1084void ucol_putOptionsToHeader(UCollator* result, UColOptionSet * opts, UErrorCode *status); 1085#endif 1086 1087U_CFUNC 1088void ucol_updateInternalState(UCollator *coll, UErrorCode *status); 1089 1090U_CFUNC uint32_t U_EXPORT2 ucol_getFirstCE(const UCollator *coll, UChar u, UErrorCode *status); 1091U_CAPI UBool U_EXPORT2 ucol_isTailored(const UCollator *coll, const UChar u, UErrorCode *status); 1092 1093U_CAPI const InverseUCATableHeader* U_EXPORT2 ucol_initInverseUCA(UErrorCode *status); 1094 1095U_CAPI void U_EXPORT2 1096uprv_uca_initImplicitConstants(UErrorCode *status); 1097 1098U_CAPI uint32_t U_EXPORT2 1099uprv_uca_getImplicitFromRaw(UChar32 cp); 1100 1101/*U_CFUNC uint32_t U_EXPORT2 1102uprv_uca_getImplicitPrimary(UChar32 cp);*/ 1103 1104U_CAPI UChar32 U_EXPORT2 1105uprv_uca_getRawFromImplicit(uint32_t implicit); 1106 1107U_CAPI UChar32 U_EXPORT2 1108uprv_uca_getRawFromCodePoint(UChar32 i); 1109 1110U_CAPI UChar32 U_EXPORT2 1111uprv_uca_getCodePointFromRaw(UChar32 i); 1112 1113typedef const UChar* GetCollationRulesFunction(void* context, const char* locale, const char* type, int32_t* pLength, UErrorCode* status); 1114 1115U_CAPI UCollator* U_EXPORT2 1116ucol_openRulesForImport( const UChar *rules, 1117 int32_t rulesLength, 1118 UColAttributeValue normalizationMode, 1119 UCollationStrength strength, 1120 UParseError *parseError, 1121 GetCollationRulesFunction importFunc, 1122 void* context, 1123 UErrorCode *status); 1124 1125 1126U_CFUNC void U_EXPORT2 1127ucol_buildPermutationTable(UCollator *coll, UErrorCode *status); 1128 1129U_CFUNC int U_EXPORT2 1130ucol_getLeadBytesForReorderCode(const UCollator *uca, int reorderCode, uint16_t* returnLeadBytes, int returnCapacity); 1131 1132U_CFUNC int U_EXPORT2 1133ucol_getReorderCodesForLeadByte(const UCollator *uca, int leadByte, int16_t* returnReorderCodes, int returnCapacity); 1134 1135#ifdef XP_CPLUSPLUS 1136/* 1137 * Test whether a character is potentially "unsafe" for use as a collation 1138 * starting point. Unsafe chars are those with combining class != 0 plus 1139 * those that are the 2nd thru nth character in a contraction sequence. 1140 * 1141 * Function is in header file because it's used in both collation and string search, 1142 * and needs to be inline for performance. 1143 */ 1144static inline UBool ucol_unsafeCP(UChar c, const UCollator *coll) { 1145 int32_t hash; 1146 uint8_t htbyte; 1147 1148 if (c < coll->minUnsafeCP) { 1149 return FALSE; 1150 } 1151 1152 hash = c; 1153 if (hash >= UCOL_UNSAFECP_TABLE_SIZE*8) { 1154 if(UTF_IS_SURROGATE(c)) { 1155 /* Lead or trail surrogate */ 1156 /* These are always considered unsafe. */ 1157 return TRUE; 1158 } 1159 hash = (hash & UCOL_UNSAFECP_TABLE_MASK) + 256; 1160 } 1161 htbyte = coll->unsafeCP[hash>>3]; 1162 return ((htbyte >> (hash & 7)) & 1); 1163} 1164#endif /* XP_CPLUSPLUS */ 1165 1166/* The offsetBuffer in collIterate might need to be freed to avoid memory leaks. */ 1167void ucol_freeOffsetBuffer(U_NAMESPACE_QUALIFIER collIterate *s); 1168 1169#endif /* #if !UCONFIG_NO_COLLATION */ 1170 1171#endif 1172