10596faeddefbf198de137d5e893708495ab1584cFredrik Roubert// © 2016 and later: Unicode, Inc. and others. 264339d36f8bd4db5025fe2988eda22b491a9219cFredrik Roubert// License & terms of use: http://www.unicode.org/copyright.html 3fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius/* 4fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius******************************************************************************* 51b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert* Copyright (C) 2013-2015, International Business Machines 6fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius* Corporation and others. All Rights Reserved. 7fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius******************************************************************************* 8fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius* collationfastlatin.h 9fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius* 10fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius* created on: 2013aug09 11fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius* created by: Markus W. Scherer 12fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius*/ 13fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 14fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#ifndef __COLLATIONFASTLATIN_H__ 15fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#define __COLLATIONFASTLATIN_H__ 16fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 17fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "unicode/utypes.h" 18fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 19fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#if !UCONFIG_NO_COLLATION 20fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 21fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusU_NAMESPACE_BEGIN 22fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 23fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusstruct CollationData; 24fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusstruct CollationSettings; 25fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 26fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusclass U_I18N_API CollationFastLatin /* all static */ { 27fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliuspublic: 28fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius /** 29fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Fast Latin format version (one byte 1..FF). 30fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Must be incremented for any runtime-incompatible changes, 31fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * in particular, for changes to any of the following constants. 32fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * 33fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * When the major version number of the main data format changes, 34fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * we can reset this fast Latin version to 1. 35fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius */ 361b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert static const uint16_t VERSION = 2; 37fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 38fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const int32_t LATIN_MAX = 0x17f; 39fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const int32_t LATIN_LIMIT = LATIN_MAX + 1; 40fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 41fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const int32_t LATIN_MAX_UTF8_LEAD = 0xc5; // UTF-8 lead byte of LATIN_MAX 42fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 43fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const int32_t PUNCT_START = 0x2000; 44fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const int32_t PUNCT_LIMIT = 0x2040; 45fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 46fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // excludes U+FFFE & U+FFFF 47fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const int32_t NUM_FAST_CHARS = LATIN_LIMIT + (PUNCT_LIMIT - PUNCT_START); 48fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 49fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Note on the supported weight ranges: 50fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Analysis of UCA 6.3 and CLDR 23 non-search tailorings shows that 51fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // the CEs for characters in the above ranges, excluding expansions with length >2, 52fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // excluding contractions of >2 characters, and other restrictions 53fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // (see the builder's getCEsFromCE32()), 54fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // use at most about 150 primary weights, 55fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // where about 94 primary weights are possibly-variable (space/punct/symbol/currency), 56fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // at most 4 secondary before-common weights, 57fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // at most 4 secondary after-common weights, 58fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // at most 16 secondary high weights (in secondary CEs), and 59fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // at most 4 tertiary after-common weights. 60fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // The following ranges are designed to support slightly more weights than that. 61fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // (en_US_POSIX is unusual: It creates about 64 variable + 116 Latin primaries.) 62fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 63fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Digits may use long primaries (preserving more short ones) 64fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // or short primaries (faster) without changing this data structure. 65fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // (If we supported numeric collation, then digits would have to have long primaries 66fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // so that special handling does not affect the fast path.) 67fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 68fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t SHORT_PRIMARY_MASK = 0xfc00; // bits 15..10 69fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t INDEX_MASK = 0x3ff; // bits 9..0 for expansions & contractions 70fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t SECONDARY_MASK = 0x3e0; // bits 9..5 71fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t CASE_MASK = 0x18; // bits 4..3 72fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t LONG_PRIMARY_MASK = 0xfff8; // bits 15..3 73fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t TERTIARY_MASK = 7; // bits 2..0 74fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t CASE_AND_TERTIARY_MASK = CASE_MASK | TERTIARY_MASK; 75fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 76fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t TWO_SHORT_PRIMARIES_MASK = 77fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius (SHORT_PRIMARY_MASK << 16) | SHORT_PRIMARY_MASK; // 0xfc00fc00 78fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t TWO_LONG_PRIMARIES_MASK = 79fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius (LONG_PRIMARY_MASK << 16) | LONG_PRIMARY_MASK; // 0xfff8fff8 80fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t TWO_SECONDARIES_MASK = 81fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius (SECONDARY_MASK << 16) | SECONDARY_MASK; // 0x3e003e0 82fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t TWO_CASES_MASK = 83fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius (CASE_MASK << 16) | CASE_MASK; // 0x180018 84fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t TWO_TERTIARIES_MASK = 85fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius (TERTIARY_MASK << 16) | TERTIARY_MASK; // 0x70007 86fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 87fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius /** 88fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Contraction with one fast Latin character. 89fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Use INDEX_MASK to find the start of the contraction list after the fixed table. 90fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * The first entry contains the default mapping. 91fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Otherwise use CONTR_CHAR_MASK for the contraction character index 92fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * (in ascending order). 93fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Use CONTR_LENGTH_SHIFT for the length of the entry 94fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * (1=BAIL_OUT, 2=one CE, 3=two CEs). 95fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * 96fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Also, U+0000 maps to a contraction entry, so that the fast path need not 97fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * check for NUL termination. 98fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * It usually maps to a contraction list with only the completely ignorable default value. 99fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius */ 100fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t CONTRACTION = 0x400; 101fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius /** 102fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * An expansion encodes two CEs. 103fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Use INDEX_MASK to find the pair of CEs after the fixed table. 104fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * 105fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * The higher a mini CE value, the easier it is to process. 106fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * For expansions and higher, no context needs to be considered. 107fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius */ 108fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t EXPANSION = 0x800; 109fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius /** 110fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Encodes one CE with a long/low mini primary (there are 128). 111fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * All potentially-variable primaries must be in this range, 112fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * to make the short-primary path as fast as possible. 113fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius */ 114fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t MIN_LONG = 0xc00; 115fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t LONG_INC = 8; 116fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t MAX_LONG = 0xff8; 117fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius /** 118fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Encodes one CE with a short/high primary (there are 60), 119fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * plus a secondary CE if the secondary weight is high. 120fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Fast handling: At least all letter primaries should be in this range. 121fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius */ 122fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t MIN_SHORT = 0x1000; 123fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t SHORT_INC = 0x400; 124fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius /** The highest primary weight is reserved for U+FFFF. */ 125fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t MAX_SHORT = SHORT_PRIMARY_MASK; 126fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 127fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t MIN_SEC_BEFORE = 0; // must add SEC_OFFSET 128fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t SEC_INC = 0x20; 129fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t MAX_SEC_BEFORE = MIN_SEC_BEFORE + 4 * SEC_INC; // 5 before common 130fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t COMMON_SEC = MAX_SEC_BEFORE + SEC_INC; 131fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t MIN_SEC_AFTER = COMMON_SEC + SEC_INC; 132fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t MAX_SEC_AFTER = MIN_SEC_AFTER + 5 * SEC_INC; // 6 after common 133fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t MIN_SEC_HIGH = MAX_SEC_AFTER + SEC_INC; // 20 high secondaries 134fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t MAX_SEC_HIGH = SECONDARY_MASK; 135fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 136fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius /** 137fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Lookup: Add this offset to secondary weights, except for completely ignorable CEs. 138fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Must be greater than any special value, e.g., MERGE_WEIGHT. 139fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * The exact value is not relevant for the format version. 140fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius */ 141fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t SEC_OFFSET = SEC_INC; 142fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t COMMON_SEC_PLUS_OFFSET = COMMON_SEC + SEC_OFFSET; 143fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 144fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t TWO_SEC_OFFSETS = 145fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius (SEC_OFFSET << 16) | SEC_OFFSET; // 0x200020 146fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t TWO_COMMON_SEC_PLUS_OFFSET = 147fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius (COMMON_SEC_PLUS_OFFSET << 16) | COMMON_SEC_PLUS_OFFSET; 148fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 149fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t LOWER_CASE = 8; // case bits include this offset 150fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t TWO_LOWER_CASES = (LOWER_CASE << 16) | LOWER_CASE; // 0x80008 151fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 152fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t COMMON_TER = 0; // must add TER_OFFSET 153fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t MAX_TER_AFTER = 7; // 7 after common 154fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 155fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius /** 156fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Lookup: Add this offset to tertiary weights, except for completely ignorable CEs. 157fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Must be greater than any special value, e.g., MERGE_WEIGHT. 158fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Must be greater than case bits as well, so that with combined case+tertiary weights 159fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * plus the offset the tertiary bits does not spill over into the case bits. 160fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * The exact value is not relevant for the format version. 161fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius */ 162fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t TER_OFFSET = SEC_OFFSET; 163fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t COMMON_TER_PLUS_OFFSET = COMMON_TER + TER_OFFSET; 164fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 165fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t TWO_TER_OFFSETS = (TER_OFFSET << 16) | TER_OFFSET; 166fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t TWO_COMMON_TER_PLUS_OFFSET = 167fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius (COMMON_TER_PLUS_OFFSET << 16) | COMMON_TER_PLUS_OFFSET; 168fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 169fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t MERGE_WEIGHT = 3; 170fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t EOS = 2; // end of string 171fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t BAIL_OUT = 1; 172fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 173fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius /** 174fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Contraction result first word bits 8..0 contain the 175fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * second contraction character, as a char index 0..NUM_FAST_CHARS-1. 176fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Each contraction list is terminated with a word containing CONTR_CHAR_MASK. 177fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius */ 178fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t CONTR_CHAR_MASK = 0x1ff; 179fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius /** 180fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Contraction result first word bits 10..9 contain the result length: 181fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * 1=bail out, 2=one mini CE, 3=two mini CEs 182fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius */ 183fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const uint32_t CONTR_LENGTH_SHIFT = 9; 184fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 185fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius /** 186fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Comparison return value when the regular comparison must be used. 187fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * The exact value is not relevant for the format version. 188fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius */ 189fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static const int32_t BAIL_OUT_RESULT = -2; 190fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 191fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static inline int32_t getCharIndex(UChar c) { 192fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(c <= LATIN_MAX) { 193fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return c; 194fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(PUNCT_START <= c && c < PUNCT_LIMIT) { 195fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return c - (PUNCT_START - LATIN_LIMIT); 196fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 197fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Not a fast Latin character. 198fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Note: U+FFFE & U+FFFF are forbidden in tailorings 199fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // and thus do not occur in any contractions. 200fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return -1; 201fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 202fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 203fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 204fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius /** 205fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Computes the options value for the compare functions 206fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * and writes the precomputed primary weights. 207fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Returns -1 if the Latin fastpath is not supported for the data and settings. 208fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * The capacity must be LATIN_LIMIT. 209fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius */ 210fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static int32_t getOptions(const CollationData *data, const CollationSettings &settings, 211fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint16_t *primaries, int32_t capacity); 212fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 213fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static int32_t compareUTF16(const uint16_t *table, const uint16_t *primaries, int32_t options, 214fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius const UChar *left, int32_t leftLength, 215fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius const UChar *right, int32_t rightLength); 216fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 217fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static int32_t compareUTF8(const uint16_t *table, const uint16_t *primaries, int32_t options, 218fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius const uint8_t *left, int32_t leftLength, 219fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius const uint8_t *right, int32_t rightLength); 220fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 221fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusprivate: 222fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static uint32_t lookup(const uint16_t *table, UChar32 c); 223fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static uint32_t lookupUTF8(const uint16_t *table, UChar32 c, 224fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius const uint8_t *s8, int32_t &sIndex, int32_t sLength); 225fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static uint32_t lookupUTF8Unsafe(const uint16_t *table, UChar32 c, 226fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius const uint8_t *s8, int32_t &sIndex); 227fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 228fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static uint32_t nextPair(const uint16_t *table, UChar32 c, uint32_t ce, 229fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius const UChar *s16, const uint8_t *s8, int32_t &sIndex, int32_t &sLength); 230fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 231fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static inline uint32_t getPrimaries(uint32_t variableTop, uint32_t pair) { 232fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t ce = pair & 0xffff; 233fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ce >= MIN_SHORT) { return pair & TWO_SHORT_PRIMARIES_MASK; } 234fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ce > variableTop) { return pair & TWO_LONG_PRIMARIES_MASK; } 235fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ce >= MIN_LONG) { return 0; } // variable 236fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return pair; // special mini CE 237fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 238fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static inline uint32_t getSecondariesFromOneShortCE(uint32_t ce) { 239fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce &= SECONDARY_MASK; 240fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ce < MIN_SEC_HIGH) { 241fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return ce + SEC_OFFSET; 242fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 243fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return ((ce + SEC_OFFSET) << 16) | COMMON_SEC_PLUS_OFFSET; 244fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 245fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 246fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static uint32_t getSecondaries(uint32_t variableTop, uint32_t pair); 247fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static uint32_t getCases(uint32_t variableTop, UBool strengthIsPrimary, uint32_t pair); 248fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static uint32_t getTertiaries(uint32_t variableTop, UBool withCaseBits, uint32_t pair); 249fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius static uint32_t getQuaternaries(uint32_t variableTop, uint32_t pair); 250fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 251fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusprivate: 252fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius CollationFastLatin(); // no constructor 253fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius}; 254fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 255fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius/* 256fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Format of the CollationFastLatin data table. 2571b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert * CollationFastLatin::VERSION = 2. 258fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * 259fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * This table contains data for a Latin-text collation fastpath. 260fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * The data is stored as an array of uint16_t which contains the following parts. 261fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * 262fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * uint16_t -- version & header length 263fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Bits 15..8: version, must match the VERSION 264fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * 7..0: length of the header 265fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * 266fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * uint16_t varTops[header length - 1] 2671b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert * Version 2: 2681b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert * varTops[m] is the highest CollationFastLatin long-primary weight 2691b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert * of supported maxVariable group m 2701b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert * (special reorder group space, punct, symbol, currency). 2711b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert * 2721b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert * Version 1: 273fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Each of these values maps the variable top lead byte of a supported maxVariable group 274fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * to the highest CollationFastLatin long-primary weight. 275fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * The values are stored in ascending order. 276fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Bits 15..7: max fast-Latin long-primary weight (bits 11..3 shifted left by 4 bits) 277fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * 6..0: regular primary lead byte 278fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * 279fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * uint16_t miniCEs[0x1c0] 280fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * A mini collation element for each character U+0000..U+017F and U+2000..U+203F. 281fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Each value encodes one or two mini CEs (two are possible if the first one 282fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * has a short mini primary and the second one is a secondary CE, i.e., primary == 0), 283fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * or points to an expansion or to a contraction table. 284fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * U+0000 always has a contraction entry, 285fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * so that NUL-termination need not be tested in the fastpath. 286fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * If the collation elements for a character or contraction cannot be encoded in this format, 287fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * then the BAIL_OUT value is stored. 288fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * For details see the comments for the class constants. 289fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * 290fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * uint16_t expansions[variable length]; 291fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Expansion mini CEs contain an offset relative to just after the miniCEs table. 292fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * An expansions contains exactly 2 mini CEs. 293fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * 294fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * uint16_t contractions[variable length]; 295fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Contraction mini CEs contain an offset relative to just after the miniCEs table. 296fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * It points to a list of tuples which map from a contraction suffix character to a result. 297fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * First uint16_t of each tuple: 298fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Bits 10..9: Length of the result (1..3), see comments on CONTR_LENGTH_SHIFT. 299fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Bits 8..0: Contraction character, see comments on CONTR_CHAR_MASK. 300fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * This is followed by 0, 1, or 2 uint16_t according to the length. 301fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Each list is terminated by an entry with CONTR_CHAR_MASK. 302fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Each list starts with such an entry which also contains the default result 303fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * for when there is no contraction match. 3041b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert * 3051b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert * ----------------- 3061b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert * Changes for version 2 (ICU 55) 3071b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert * 3081b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert * Special reorder groups do not necessarily start on whole primary lead bytes any more. 3091b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert * Therefore, the varTops data has a new format: 3101b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert * Version 1 stored the lead bytes of the highest root primaries for 3111b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert * the maxVariable-supported special reorder groups. 3121b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert * Now the top 16 bits would need to be stored, 3131b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert * and it is simpler to store only the fast-Latin weights. 314fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius */ 315fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 316fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusU_NAMESPACE_END 317fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 318fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#endif // !UCONFIG_NO_COLLATION 319fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#endif // __COLLATIONFASTLATIN_H__ 320