10596faeddefbf198de137d5e893708495ab1584cFredrik Roubert// © 2016 and later: Unicode, Inc. and others. 264339d36f8bd4db5025fe2988eda22b491a9219cFredrik Roubert// License & terms of use: http://www.unicode.org/copyright.html 3fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius/* 4fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius******************************************************************************* 51b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert* Copyright (C) 2013-2015, International Business Machines 6fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius* Corporation and others. All Rights Reserved. 7fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius******************************************************************************* 8fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius* collationfastlatinbuilder.cpp 9fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius* 10fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius* created on: 2013aug09 11fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius* created by: Markus W. Scherer 12fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius*/ 13fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 14fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#define DEBUG_COLLATION_FAST_LATIN_BUILDER 0 // 0 or 1 or 2 15fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#if DEBUG_COLLATION_FAST_LATIN_BUILDER 16fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include <stdio.h> 17fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include <string> 18fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#endif 19fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 20fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "unicode/utypes.h" 21fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 22fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#if !UCONFIG_NO_COLLATION 23fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 24fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "unicode/ucol.h" 25fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "unicode/ucharstrie.h" 26fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "unicode/unistr.h" 27fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "unicode/uobject.h" 28fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "unicode/uscript.h" 29fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "cmemory.h" 30fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "collation.h" 31fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "collationdata.h" 32fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "collationfastlatin.h" 33fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "collationfastlatinbuilder.h" 34fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "uassert.h" 35fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "uvectr64.h" 36fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 37fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusU_NAMESPACE_BEGIN 38fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 39fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusstruct CollationData; 40fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 41fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusnamespace { 42fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 43fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius/** 44fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Compare two signed int64_t values as if they were unsigned. 45fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius */ 46fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusint32_t 47fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliuscompareInt64AsUnsigned(int64_t a, int64_t b) { 48fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if((uint64_t)a < (uint64_t)b) { 49fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return -1; 50fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if((uint64_t)a > (uint64_t)b) { 51fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return 1; 52fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 53fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return 0; 54fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 55fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 56fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 57fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius// TODO: Merge this with the near-identical version in collationbasedatabuilder.cpp 58fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius/** 59fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Like Java Collections.binarySearch(List, String, Comparator). 60fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * 61fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * @return the index>=0 where the item was found, 62fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * or the index<0 for inserting the string at ~index in sorted order 63fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius */ 64fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusint32_t 65fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusbinarySearch(const int64_t list[], int32_t limit, int64_t ce) { 66fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if (limit == 0) { return ~0; } 67fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t start = 0; 68fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for (;;) { 69fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t i = (start + limit) / 2; 70fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t cmp = compareInt64AsUnsigned(ce, list[i]); 71fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if (cmp == 0) { 72fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return i; 73fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if (cmp < 0) { 74fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if (i == start) { 75fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return ~start; // insert ce before i 76fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 77fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius limit = i; 78fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 79fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if (i == start) { 80fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return ~(start + 1); // insert ce after i 81fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 82fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius start = i; 83fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 84fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 85fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 86fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 87fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} // namespace 88fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 89fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::CollationFastLatinBuilder(UErrorCode &errorCode) 90fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius : ce0(0), ce1(0), 91fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius contractionCEs(errorCode), uniqueCEs(errorCode), 92fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCEs(NULL), 93fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius firstDigitPrimary(0), firstLatinPrimary(0), lastLatinPrimary(0), 94fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius firstShortPrimary(0), shortPrimaryOverflow(FALSE), 95fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius headerLength(0) { 96fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 97fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 98fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::~CollationFastLatinBuilder() { 99fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uprv_free(miniCEs); 100fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 101fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 102fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusUBool 103fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::forData(const CollationData &data, UErrorCode &errorCode) { 104fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_FAILURE(errorCode)) { return FALSE; } 105fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(!result.isEmpty()) { // This builder is not reusable. 106fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius errorCode = U_INVALID_STATE_ERROR; 107fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 108fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 109fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(!loadGroups(data, errorCode)) { return FALSE; } 110fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 111fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Fast handling of digits. 112fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius firstShortPrimary = firstDigitPrimary; 113fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius getCEs(data, errorCode); 114fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(!encodeUniqueCEs(errorCode)) { return FALSE; } 115fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(shortPrimaryOverflow) { 116fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Give digits long mini primaries, 117fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // so that there are more short primaries for letters. 118fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius firstShortPrimary = firstLatinPrimary; 119fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius resetCEs(); 120fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius getCEs(data, errorCode); 121fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(!encodeUniqueCEs(errorCode)) { return FALSE; } 122fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 123fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Note: If we still have a short-primary overflow but not a long-primary overflow, 124fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // then we could calculate how many more long primaries would fit, 125fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // and set the firstShortPrimary to that many after the current firstShortPrimary, 126fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // and try again. 127fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // However, this might only benefit the en_US_POSIX tailoring, 128fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // and it is simpler to suppress building fast Latin data for it in genrb, 129fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // or by returning FALSE here if shortPrimaryOverflow. 130fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 131fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UBool ok = !shortPrimaryOverflow && 132fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius encodeCharCEs(errorCode) && encodeContractions(errorCode); 133fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius contractionCEs.removeAllElements(); // might reduce heap memory usage 134fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uniqueCEs.removeAllElements(); 135fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return ok; 136fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 137fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 138fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusUBool 139fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::loadGroups(const CollationData &data, UErrorCode &errorCode) { 140fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_FAILURE(errorCode)) { return FALSE; } 1411b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert headerLength = 1 + NUM_SPECIAL_GROUPS; 1421b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert uint32_t r0 = (CollationFastLatin::VERSION << 8) | headerLength; 1431b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert result.append((UChar)r0); 144fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // The first few reordering groups should be special groups 145fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // (space, punct, ..., digit) followed by Latn, then Grek and other scripts. 1461b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert for(int32_t i = 0; i < NUM_SPECIAL_GROUPS; ++i) { 1471b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert lastSpecialPrimaries[i] = data.getLastPrimaryForGroup(UCOL_REORDER_CODE_FIRST + i); 1481b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert if(lastSpecialPrimaries[i] == 0) { 1491b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert // missing data 150fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 151fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 15264339d36f8bd4db5025fe2988eda22b491a9219cFredrik Roubert result.append((UChar)0); // reserve a slot for this group 1531b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert } 1541b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert 1551b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert firstDigitPrimary = data.getFirstPrimaryForGroup(UCOL_REORDER_CODE_DIGIT); 1561b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert firstLatinPrimary = data.getFirstPrimaryForGroup(USCRIPT_LATIN); 1571b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert lastLatinPrimary = data.getLastPrimaryForGroup(USCRIPT_LATIN); 1581b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert if(firstDigitPrimary == 0 || firstLatinPrimary == 0) { 1591b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert // missing data 1601b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert return FALSE; 161fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 162fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return TRUE; 163fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 164fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 165fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusUBool 166fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::inSameGroup(uint32_t p, uint32_t q) const { 167fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Both or neither need to be encoded as short primaries, 168fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // so that we can test only one and use the same bit mask. 169fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(p >= firstShortPrimary) { 170fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return q >= firstShortPrimary; 171fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(q >= firstShortPrimary) { 172fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 173fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 174fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Both or neither must be potentially-variable, 175fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // so that we can test only one and determine if both are variable. 1761b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert uint32_t lastVariablePrimary = lastSpecialPrimaries[NUM_SPECIAL_GROUPS - 1]; 1771b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert if(p > lastVariablePrimary) { 1781b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert return q > lastVariablePrimary; 1791b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert } else if(q > lastVariablePrimary) { 180fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 181fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 182fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Both will be encoded with long mini primaries. 183fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // They must be in the same special reordering group, 184fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // so that we can test only one and determine if both are variable. 185fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(p != 0 && q != 0); 1861b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert for(int32_t i = 0;; ++i) { // will terminate 1871b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert uint32_t lastPrimary = lastSpecialPrimaries[i]; 1881b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert if(p <= lastPrimary) { 1891b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert return q <= lastPrimary; 1901b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert } else if(q <= lastPrimary) { 191fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 192fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 193fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 194fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 195fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 196fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusvoid 197fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::resetCEs() { 198fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius contractionCEs.removeAllElements(); 199fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uniqueCEs.removeAllElements(); 200fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius shortPrimaryOverflow = FALSE; 201fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.truncate(headerLength); 202fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 203fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 204fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusvoid 205fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::getCEs(const CollationData &data, UErrorCode &errorCode) { 206fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_FAILURE(errorCode)) { return; } 207fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t i = 0; 208fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(UChar c = 0;; ++i, ++c) { 209fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(c == CollationFastLatin::LATIN_LIMIT) { 210fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius c = CollationFastLatin::PUNCT_START; 211fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(c == CollationFastLatin::PUNCT_LIMIT) { 212fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 213fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 214fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius const CollationData *d; 215fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t ce32 = data.getCE32(c); 216fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ce32 == Collation::FALLBACK_CE32) { 217fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius d = data.base; 218fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = d->getCE32(c); 219fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 220fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius d = &data; 221fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 222fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(getCEsFromCE32(*d, c, ce32, errorCode)) { 223fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius charCEs[i][0] = ce0; 224fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius charCEs[i][1] = ce1; 225fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addUniqueCE(ce0, errorCode); 226fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addUniqueCE(ce1, errorCode); 227fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 228fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // bail out for c 229fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius charCEs[i][0] = ce0 = Collation::NO_CE; 230fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius charCEs[i][1] = ce1 = 0; 231fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 232fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(c == 0 && !isContractionCharCE(ce0)) { 233fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Always map U+0000 to a contraction. 234fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Write a contraction list with only a default value if there is no real contraction. 235fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(contractionCEs.isEmpty()); 236fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addContractionEntry(CollationFastLatin::CONTR_CHAR_MASK, ce0, ce1, errorCode); 237fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius charCEs[0][0] = ((int64_t)Collation::NO_CE_PRIMARY << 32) | CONTRACTION_FLAG; 238fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius charCEs[0][1] = 0; 239fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 240fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 241fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Terminate the last contraction list. 242fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius contractionCEs.addElement(CollationFastLatin::CONTR_CHAR_MASK, errorCode); 243fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 244fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 245fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusUBool 246fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::getCEsFromCE32(const CollationData &data, UChar32 c, uint32_t ce32, 247fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UErrorCode &errorCode) { 248fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_FAILURE(errorCode)) { return FALSE; } 249fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = data.getFinalCE32(ce32); 250fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce1 = 0; 251fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(Collation::isSimpleOrLongCE32(ce32)) { 252fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce0 = Collation::ceFromCE32(ce32); 253fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 254fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius switch(Collation::tagFromCE32(ce32)) { 255fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::LATIN_EXPANSION_TAG: 256fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce0 = Collation::latinCE0FromCE32(ce32); 257fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce1 = Collation::latinCE1FromCE32(ce32); 258fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 259fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::EXPANSION32_TAG: { 260fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius const uint32_t *ce32s = data.ce32s + Collation::indexFromCE32(ce32); 261fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t length = Collation::lengthFromCE32(ce32); 262fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(length <= 2) { 263fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce0 = Collation::ceFromCE32(ce32s[0]); 264fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(length == 2) { 265fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce1 = Collation::ceFromCE32(ce32s[1]); 266fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 267fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 268fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 269fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 270fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 271fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 272fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::EXPANSION_TAG: { 273fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius const int64_t *ces = data.ces + Collation::indexFromCE32(ce32); 274fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t length = Collation::lengthFromCE32(ce32); 275fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(length <= 2) { 276fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce0 = ces[0]; 277fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(length == 2) { 278fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce1 = ces[1]; 279fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 280fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 281fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 282fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 283fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 284fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 285fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Note: We could support PREFIX_TAG (assert c>=0) 286fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // by recursing on its default CE32 and checking that none of the prefixes starts 287fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // with a fast Latin character. 288fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // However, currently (2013) there are only the L-before-middle-dot 289fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // prefix mappings in the Latin range, and those would be rejected anyway. 290fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::CONTRACTION_TAG: 291fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(c >= 0); 292fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return getCEsFromContractionCE32(data, ce32, errorCode); 293fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::OFFSET_TAG: 294fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(c >= 0); 295fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce0 = data.getCEFromOffsetCE32(c, ce32); 296fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 297fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius default: 298fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 299fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 300fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 301fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // A mapping can be completely ignorable. 302fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ce0 == 0) { return ce1 == 0; } 303fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // We do not support an ignorable ce0 unless it is completely ignorable. 304fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t p0 = (uint32_t)(ce0 >> 32); 305fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(p0 == 0) { return FALSE; } 306fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // We only support primaries up to the Latin script. 307fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(p0 > lastLatinPrimary) { return FALSE; } 308fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // We support non-common secondary and case weights only together with short primaries. 309fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t lower32_0 = (uint32_t)ce0; 310fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(p0 < firstShortPrimary) { 311fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t sc0 = lower32_0 & Collation::SECONDARY_AND_CASE_MASK; 312fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(sc0 != Collation::COMMON_SECONDARY_CE) { return FALSE; } 313fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 314fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // No below-common tertiary weights. 315fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if((lower32_0 & Collation::ONLY_TERTIARY_MASK) < Collation::COMMON_WEIGHT16) { return FALSE; } 316fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ce1 != 0) { 317fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Both primaries must be in the same group, 318fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // or both must get short mini primaries, 319fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // or a short-primary CE is followed by a secondary CE. 320fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // This is so that we can test the first primary and use the same mask for both, 321fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // and determine for both whether they are variable. 322fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t p1 = (uint32_t)(ce1 >> 32); 323fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(p1 == 0 ? p0 < firstShortPrimary : !inSameGroup(p0, p1)) { return FALSE; } 324fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t lower32_1 = (uint32_t)ce1; 325fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // No tertiary CEs. 326fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if((lower32_1 >> 16) == 0) { return FALSE; } 327fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // We support non-common secondary and case weights 328fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // only for secondary CEs or together with short primaries. 329fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(p1 != 0 && p1 < firstShortPrimary) { 330fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t sc1 = lower32_1 & Collation::SECONDARY_AND_CASE_MASK; 331fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(sc1 != Collation::COMMON_SECONDARY_CE) { return FALSE; } 332fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 333fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // No below-common tertiary weights. 334fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if((lower32_1 & Collation::ONLY_TERTIARY_MASK) < Collation::COMMON_WEIGHT16) { return FALSE; } 335fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 336fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // No quaternary weights. 337fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(((ce0 | ce1) & Collation::QUATERNARY_MASK) != 0) { return FALSE; } 338fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return TRUE; 339fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 340fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 341fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusUBool 342fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::getCEsFromContractionCE32(const CollationData &data, uint32_t ce32, 343fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UErrorCode &errorCode) { 344fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_FAILURE(errorCode)) { return FALSE; } 345fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius const UChar *p = data.contexts + Collation::indexFromCE32(ce32); 346fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = CollationData::readCE32(p); // Default if no suffix match. 347fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Since the original ce32 is not a prefix mapping, 348fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // the default ce32 must not be another contraction. 349fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(!Collation::isContractionCE32(ce32)); 350fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t contractionIndex = contractionCEs.size(); 351fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(getCEsFromCE32(data, U_SENTINEL, ce32, errorCode)) { 352fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addContractionEntry(CollationFastLatin::CONTR_CHAR_MASK, ce0, ce1, errorCode); 353fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 354fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Bail out for c-without-contraction. 355fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addContractionEntry(CollationFastLatin::CONTR_CHAR_MASK, Collation::NO_CE, 0, errorCode); 356fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 357fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Handle an encodable contraction unless the next contraction is too long 358fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // and starts with the same character. 359fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t prevX = -1; 360fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UBool addContraction = FALSE; 361fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UCharsTrie::Iterator suffixes(p + 2, 0, errorCode); 362fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius while(suffixes.next(errorCode)) { 363fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius const UnicodeString &suffix = suffixes.getString(); 364fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t x = CollationFastLatin::getCharIndex(suffix.charAt(0)); 365fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(x < 0) { continue; } // ignore anything but fast Latin text 366fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(x == prevX) { 367fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(addContraction) { 368fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Bail out for all contractions starting with this character. 369fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addContractionEntry(x, Collation::NO_CE, 0, errorCode); 370fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addContraction = FALSE; 371fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 372fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius continue; 373fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 374fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(addContraction) { 375fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addContractionEntry(prevX, ce0, ce1, errorCode); 376fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 377fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = (uint32_t)suffixes.getValue(); 378fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(suffix.length() == 1 && getCEsFromCE32(data, U_SENTINEL, ce32, errorCode)) { 379fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addContraction = TRUE; 380fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 381fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addContractionEntry(x, Collation::NO_CE, 0, errorCode); 382fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addContraction = FALSE; 383fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 384fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius prevX = x; 385fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 386fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(addContraction) { 387fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addContractionEntry(prevX, ce0, ce1, errorCode); 388fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 389fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_FAILURE(errorCode)) { return FALSE; } 390fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Note: There might not be any fast Latin contractions, but 391fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // we need to enter contraction handling anyway so that we can bail out 392fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // when there is a non-fast-Latin character following. 393fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // For example: Danish &Y<<u+umlaut, when we compare Y vs. u\u0308 we need to see the 394fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // following umlaut and bail out, rather than return the difference of Y vs. u. 395fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce0 = ((int64_t)Collation::NO_CE_PRIMARY << 32) | CONTRACTION_FLAG | contractionIndex; 396fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce1 = 0; 397fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return TRUE; 398fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 399fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 400fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusvoid 401fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::addContractionEntry(int32_t x, int64_t cce0, int64_t cce1, 402fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UErrorCode &errorCode) { 403fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius contractionCEs.addElement(x, errorCode); 404fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius contractionCEs.addElement(cce0, errorCode); 405fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius contractionCEs.addElement(cce1, errorCode); 406fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addUniqueCE(cce0, errorCode); 407fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addUniqueCE(cce1, errorCode); 408fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 409fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 410fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusvoid 411fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::addUniqueCE(int64_t ce, UErrorCode &errorCode) { 412fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_FAILURE(errorCode)) { return; } 413fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ce == 0 || (uint32_t)(ce >> 32) == Collation::NO_CE_PRIMARY) { return; } 414fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce &= ~(int64_t)Collation::CASE_MASK; // blank out case bits 415fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t i = binarySearch(uniqueCEs.getBuffer(), uniqueCEs.size(), ce); 416fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(i < 0) { 417fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uniqueCEs.insertElementAt(ce, ~i, errorCode); 418fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 419fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 420fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 421fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusuint32_t 422fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::getMiniCE(int64_t ce) const { 423fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce &= ~(int64_t)Collation::CASE_MASK; // blank out case bits 424fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t index = binarySearch(uniqueCEs.getBuffer(), uniqueCEs.size(), ce); 425fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(index >= 0); 426fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return miniCEs[index]; 427fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 428fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 429fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusUBool 430fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::encodeUniqueCEs(UErrorCode &errorCode) { 431fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_FAILURE(errorCode)) { return FALSE; } 432fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uprv_free(miniCEs); 433fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCEs = (uint16_t *)uprv_malloc(uniqueCEs.size() * 2); 434fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(miniCEs == NULL) { 435fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius errorCode = U_MEMORY_ALLOCATION_ERROR; 436fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 437fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 4381b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert int32_t group = 0; 4391b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert uint32_t lastGroupPrimary = lastSpecialPrimaries[group]; 440fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // The lowest unique CE must be at least a secondary CE. 441fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(((uint32_t)uniqueCEs.elementAti(0) >> 16) != 0); 442fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t prevPrimary = 0; 443fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t prevSecondary = 0; 444fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t pri = 0; 445fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t sec = 0; 446fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t ter = CollationFastLatin::COMMON_TER; 447fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(int32_t i = 0; i < uniqueCEs.size(); ++i) { 448fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int64_t ce = uniqueCEs.elementAti(i); 449fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Note: At least one of the p/s/t weights changes from one unique CE to the next. 450fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // (uniqueCEs does not store case bits.) 451fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t p = (uint32_t)(ce >> 32); 452fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(p != prevPrimary) { 4531b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert while(p > lastGroupPrimary) { 454fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(pri <= CollationFastLatin::MAX_LONG); 4551b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert // Set the group's header entry to the 4561b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert // last "long primary" in or before the group. 4571b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert result.setCharAt(1 + group, (UChar)pri); 4581b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert if(++group < NUM_SPECIAL_GROUPS) { 4591b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert lastGroupPrimary = lastSpecialPrimaries[group]; 460fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 4611b7d32f919554dda9c193b32188251337bc756f1Fredrik Roubert lastGroupPrimary = 0xffffffff; 462fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 463fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 464fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 465fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(p < firstShortPrimary) { 466fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(pri == 0) { 467fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius pri = CollationFastLatin::MIN_LONG; 468fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(pri < CollationFastLatin::MAX_LONG) { 469fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius pri += CollationFastLatin::LONG_INC; 470fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 471fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#if DEBUG_COLLATION_FAST_LATIN_BUILDER 472fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius printf("long-primary overflow for %08x\n", p); 473fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#endif 474fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCEs[i] = CollationFastLatin::BAIL_OUT; 475fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius continue; 476fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 477fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 478fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(pri < CollationFastLatin::MIN_SHORT) { 479fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius pri = CollationFastLatin::MIN_SHORT; 480fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(pri < (CollationFastLatin::MAX_SHORT - CollationFastLatin::SHORT_INC)) { 481fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Reserve the highest primary weight for U+FFFF. 482fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius pri += CollationFastLatin::SHORT_INC; 483fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 484fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#if DEBUG_COLLATION_FAST_LATIN_BUILDER 485fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius printf("short-primary overflow for %08x\n", p); 486fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#endif 487fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius shortPrimaryOverflow = TRUE; 488fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCEs[i] = CollationFastLatin::BAIL_OUT; 489fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius continue; 490fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 491fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 492fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius prevPrimary = p; 493fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius prevSecondary = Collation::COMMON_WEIGHT16; 494fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius sec = CollationFastLatin::COMMON_SEC; 495fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ter = CollationFastLatin::COMMON_TER; 496fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 497fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t lower32 = (uint32_t)ce; 498fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t s = lower32 >> 16; 499fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(s != prevSecondary) { 500fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(pri == 0) { 501fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(sec == 0) { 502fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius sec = CollationFastLatin::MIN_SEC_HIGH; 503fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(sec < CollationFastLatin::MAX_SEC_HIGH) { 504fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius sec += CollationFastLatin::SEC_INC; 505fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 506fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCEs[i] = CollationFastLatin::BAIL_OUT; 507fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius continue; 508fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 509fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius prevSecondary = s; 510fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ter = CollationFastLatin::COMMON_TER; 511fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(s < Collation::COMMON_WEIGHT16) { 512fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(sec == CollationFastLatin::COMMON_SEC) { 513fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius sec = CollationFastLatin::MIN_SEC_BEFORE; 514fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(sec < CollationFastLatin::MAX_SEC_BEFORE) { 515fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius sec += CollationFastLatin::SEC_INC; 516fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 517fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCEs[i] = CollationFastLatin::BAIL_OUT; 518fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius continue; 519fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 520fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(s == Collation::COMMON_WEIGHT16) { 521fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius sec = CollationFastLatin::COMMON_SEC; 522fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 523fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(sec < CollationFastLatin::MIN_SEC_AFTER) { 524fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius sec = CollationFastLatin::MIN_SEC_AFTER; 525fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(sec < CollationFastLatin::MAX_SEC_AFTER) { 526fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius sec += CollationFastLatin::SEC_INC; 527fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 528fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCEs[i] = CollationFastLatin::BAIL_OUT; 529fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius continue; 530fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 531fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 532fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius prevSecondary = s; 533fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ter = CollationFastLatin::COMMON_TER; 534fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 535fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT((lower32 & Collation::CASE_MASK) == 0); // blanked out in uniqueCEs 536fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t t = lower32 & Collation::ONLY_TERTIARY_MASK; 537fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(t > Collation::COMMON_WEIGHT16) { 538fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ter < CollationFastLatin::MAX_TER_AFTER) { 539fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ++ter; 540fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 541fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCEs[i] = CollationFastLatin::BAIL_OUT; 542fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius continue; 543fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 544fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 545fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(CollationFastLatin::MIN_LONG <= pri && pri <= CollationFastLatin::MAX_LONG) { 546fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(sec == CollationFastLatin::COMMON_SEC); 547fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCEs[i] = (uint16_t)(pri | ter); 548fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 549fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCEs[i] = (uint16_t)(pri | sec | ter); 550fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 551fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 552fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#if DEBUG_COLLATION_FAST_LATIN_BUILDER 553fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius printf("last mini primary: %04x\n", pri); 554fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#endif 555fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#if DEBUG_COLLATION_FAST_LATIN_BUILDER >= 2 556fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(int32_t i = 0; i < uniqueCEs.size(); ++i) { 557fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int64_t ce = uniqueCEs.elementAti(i); 558fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius printf("unique CE 0x%016lx -> 0x%04x\n", ce, miniCEs[i]); 559fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 560fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#endif 561fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return U_SUCCESS(errorCode); 562fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 563fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 564fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusUBool 565fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::encodeCharCEs(UErrorCode &errorCode) { 566fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_FAILURE(errorCode)) { return FALSE; } 567fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t miniCEsStart = result.length(); 568fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(int32_t i = 0; i < CollationFastLatin::NUM_FAST_CHARS; ++i) { 56964339d36f8bd4db5025fe2988eda22b491a9219cFredrik Roubert result.append((UChar)0); // initialize to completely ignorable 570fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 571fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t indexBase = result.length(); 572fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(int32_t i = 0; i < CollationFastLatin::NUM_FAST_CHARS; ++i) { 573fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int64_t ce = charCEs[i][0]; 574fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(isContractionCharCE(ce)) { continue; } // defer contraction 575fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t miniCE = encodeTwoCEs(ce, charCEs[i][1]); 576fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(miniCE > 0xffff) { 577fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Note: There is a chance that this new expansion is the same as a previous one, 578fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // and if so, then we could reuse the other expansion. 579fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // However, that seems unlikely. 580fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t expansionIndex = result.length() - indexBase; 581fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(expansionIndex > (int32_t)CollationFastLatin::INDEX_MASK) { 582fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCE = CollationFastLatin::BAIL_OUT; 583fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 584fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.append((UChar)(miniCE >> 16)).append((UChar)miniCE); 585fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCE = CollationFastLatin::EXPANSION | expansionIndex; 586fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 587fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 588fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.setCharAt(miniCEsStart + i, (UChar)miniCE); 589fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 590fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return U_SUCCESS(errorCode); 591fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 592fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 593fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusUBool 594fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::encodeContractions(UErrorCode &errorCode) { 595fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // We encode all contraction lists so that the first word of a list 596fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // terminates the previous list, and we only need one additional terminator at the end. 597fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_FAILURE(errorCode)) { return FALSE; } 598fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t indexBase = headerLength + CollationFastLatin::NUM_FAST_CHARS; 599fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t firstContractionIndex = result.length(); 600fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(int32_t i = 0; i < CollationFastLatin::NUM_FAST_CHARS; ++i) { 601fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int64_t ce = charCEs[i][0]; 602fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(!isContractionCharCE(ce)) { continue; } 603fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t contractionIndex = result.length() - indexBase; 604fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(contractionIndex > (int32_t)CollationFastLatin::INDEX_MASK) { 605fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.setCharAt(headerLength + i, CollationFastLatin::BAIL_OUT); 606fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius continue; 607fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 608fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UBool firstTriple = TRUE; 609fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(int32_t index = (int32_t)ce & 0x7fffffff;; index += 3) { 610ffdc27edd5503111189fc11165c5a11289a71f79Fredrik Roubert int32_t x = static_cast<int32_t>(contractionCEs.elementAti(index)); 611fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if((uint32_t)x == CollationFastLatin::CONTR_CHAR_MASK && !firstTriple) { break; } 612fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int64_t cce0 = contractionCEs.elementAti(index + 1); 613fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int64_t cce1 = contractionCEs.elementAti(index + 2); 614fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t miniCE = encodeTwoCEs(cce0, cce1); 615fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(miniCE == CollationFastLatin::BAIL_OUT) { 616fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.append((UChar)(x | (1 << CollationFastLatin::CONTR_LENGTH_SHIFT))); 617fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(miniCE <= 0xffff) { 618fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.append((UChar)(x | (2 << CollationFastLatin::CONTR_LENGTH_SHIFT))); 619fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.append((UChar)miniCE); 620fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 621fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.append((UChar)(x | (3 << CollationFastLatin::CONTR_LENGTH_SHIFT))); 622fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.append((UChar)(miniCE >> 16)).append((UChar)miniCE); 623fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 624fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius firstTriple = FALSE; 625fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 626fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Note: There is a chance that this new contraction list is the same as a previous one, 627fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // and if so, then we could truncate the result and reuse the other list. 628fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // However, that seems unlikely. 629fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.setCharAt(headerLength + i, 630fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius (UChar)(CollationFastLatin::CONTRACTION | contractionIndex)); 631fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 632fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(result.length() > firstContractionIndex) { 633fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Terminate the last contraction list. 634fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.append((UChar)CollationFastLatin::CONTR_CHAR_MASK); 635fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 636fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(result.isBogus()) { 637fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius errorCode = U_MEMORY_ALLOCATION_ERROR; 638fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 639fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 640fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#if DEBUG_COLLATION_FAST_LATIN_BUILDER 641fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius printf("** fast Latin %d * 2 = %d bytes\n", result.length(), result.length() * 2); 642fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius puts(" header & below-digit groups map"); 643fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t i = 0; 644fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(; i < headerLength; ++i) { 645fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius printf(" %04x", result[i]); 646fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 647fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius printf("\n char mini CEs"); 648fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(CollationFastLatin::NUM_FAST_CHARS % 16 == 0); 649fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(; i < indexBase; i += 16) { 650fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UChar32 c = i - headerLength; 651fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(c >= CollationFastLatin::LATIN_LIMIT) { 652fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius c = CollationFastLatin::PUNCT_START + c - CollationFastLatin::LATIN_LIMIT; 653fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 654fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius printf("\n %04x:", c); 655fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(int32_t j = 0; j < 16; ++j) { 656fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius printf(" %04x", result[i + j]); 657fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 658fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 659fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius printf("\n expansions & contractions"); 660fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(; i < result.length(); ++i) { 661fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if((i - indexBase) % 16 == 0) { puts(""); } 662fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius printf(" %04x", result[i]); 663fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 664fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius puts(""); 665fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#endif 666fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return TRUE; 667fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 668fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 669fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusuint32_t 670fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::encodeTwoCEs(int64_t first, int64_t second) const { 671fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(first == 0) { 672fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return 0; // completely ignorable 673fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 674fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(first == Collation::NO_CE) { 675fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return CollationFastLatin::BAIL_OUT; 676fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 677fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT((uint32_t)(first >> 32) != Collation::NO_CE_PRIMARY); 678fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 679fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t miniCE = getMiniCE(first); 680fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(miniCE == CollationFastLatin::BAIL_OUT) { return miniCE; } 681fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(miniCE >= CollationFastLatin::MIN_SHORT) { 682fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Extract & copy the case bits. 683fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Shift them from normal CE bits 15..14 to mini CE bits 4..3. 684fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t c = (((uint32_t)first & Collation::CASE_MASK) >> (14 - 3)); 685fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Only in mini CEs: Ignorable case bits = 0, lowercase = 1. 686fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius c += CollationFastLatin::LOWER_CASE; 687fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCE |= c; 688fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 689fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(second == 0) { return miniCE; } 690fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 691fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t miniCE1 = getMiniCE(second); 692fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(miniCE1 == CollationFastLatin::BAIL_OUT) { return miniCE1; } 693fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 694fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t case1 = (uint32_t)second & Collation::CASE_MASK; 695fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(miniCE >= CollationFastLatin::MIN_SHORT && 696fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius (miniCE & CollationFastLatin::SECONDARY_MASK) == CollationFastLatin::COMMON_SEC) { 697fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Try to combine the two mini CEs into one. 698fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t sec1 = miniCE1 & CollationFastLatin::SECONDARY_MASK; 699fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t ter1 = miniCE1 & CollationFastLatin::TERTIARY_MASK; 700fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(sec1 >= CollationFastLatin::MIN_SEC_HIGH && case1 == 0 && 701fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ter1 == CollationFastLatin::COMMON_TER) { 702fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // sec1>=sec_high implies pri1==0. 703fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return (miniCE & ~CollationFastLatin::SECONDARY_MASK) | sec1; 704fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 705fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 706fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 707fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(miniCE1 <= CollationFastLatin::SECONDARY_MASK || CollationFastLatin::MIN_SHORT <= miniCE1) { 708fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Secondary CE, or a CE with a short primary, copy the case bits. 709fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case1 = (case1 >> (14 - 3)) + CollationFastLatin::LOWER_CASE; 710fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCE1 |= case1; 711fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 712fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return (miniCE << 16) | miniCE1; 713fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 714fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 715fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusU_NAMESPACE_END 716fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 717fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#endif // !UCONFIG_NO_COLLATION 718