1fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius/* 2fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius******************************************************************************* 3fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius* Copyright (C) 2013-2014, International Business Machines 4fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius* Corporation and others. All Rights Reserved. 5fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius******************************************************************************* 6fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius* collationfastlatinbuilder.cpp 7fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius* 8fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius* created on: 2013aug09 9fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius* created by: Markus W. Scherer 10fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius*/ 11fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 12fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#define DEBUG_COLLATION_FAST_LATIN_BUILDER 0 // 0 or 1 or 2 13fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#if DEBUG_COLLATION_FAST_LATIN_BUILDER 14fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include <stdio.h> 15fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include <string> 16fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#endif 17fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 18fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "unicode/utypes.h" 19fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 20fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#if !UCONFIG_NO_COLLATION 21fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 22fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "unicode/ucol.h" 23fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "unicode/ucharstrie.h" 24fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "unicode/unistr.h" 25fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "unicode/uobject.h" 26fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "unicode/uscript.h" 27fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "cmemory.h" 28fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "collation.h" 29fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "collationdata.h" 30fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "collationfastlatin.h" 31fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "collationfastlatinbuilder.h" 32fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "uassert.h" 33fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "uvectr64.h" 34fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 35fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusU_NAMESPACE_BEGIN 36fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 37fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusstruct CollationData; 38fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 39fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusnamespace { 40fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 41fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius/** 42fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Compare two signed int64_t values as if they were unsigned. 43fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius */ 44fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusint32_t 45fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliuscompareInt64AsUnsigned(int64_t a, int64_t b) { 46fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if((uint64_t)a < (uint64_t)b) { 47fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return -1; 48fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if((uint64_t)a > (uint64_t)b) { 49fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return 1; 50fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 51fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return 0; 52fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 53fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 54fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 55fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius// TODO: Merge this with the near-identical version in collationbasedatabuilder.cpp 56fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius/** 57fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * Like Java Collections.binarySearch(List, String, Comparator). 58fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * 59fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * @return the index>=0 where the item was found, 60fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius * or the index<0 for inserting the string at ~index in sorted order 61fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius */ 62fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusint32_t 63fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusbinarySearch(const int64_t list[], int32_t limit, int64_t ce) { 64fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if (limit == 0) { return ~0; } 65fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t start = 0; 66fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for (;;) { 67fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t i = (start + limit) / 2; 68fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t cmp = compareInt64AsUnsigned(ce, list[i]); 69fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if (cmp == 0) { 70fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return i; 71fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if (cmp < 0) { 72fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if (i == start) { 73fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return ~start; // insert ce before i 74fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 75fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius limit = i; 76fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 77fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if (i == start) { 78fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return ~(start + 1); // insert ce after i 79fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 80fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius start = i; 81fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 82fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 83fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 84fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 85fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} // namespace 86fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 87fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::CollationFastLatinBuilder(UErrorCode &errorCode) 88fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius : ce0(0), ce1(0), 89fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius contractionCEs(errorCode), uniqueCEs(errorCode), 90fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCEs(NULL), 91fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius firstDigitPrimary(0), firstLatinPrimary(0), lastLatinPrimary(0), 92fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius firstShortPrimary(0), shortPrimaryOverflow(FALSE), 93fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius headerLength(0) { 94fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 95fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 96fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::~CollationFastLatinBuilder() { 97fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uprv_free(miniCEs); 98fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 99fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 100fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusUBool 101fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::forData(const CollationData &data, UErrorCode &errorCode) { 102fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_FAILURE(errorCode)) { return FALSE; } 103fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(!result.isEmpty()) { // This builder is not reusable. 104fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius errorCode = U_INVALID_STATE_ERROR; 105fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 106fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 107fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(!loadGroups(data, errorCode)) { return FALSE; } 108fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 109fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Fast handling of digits. 110fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius firstShortPrimary = firstDigitPrimary; 111fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius getCEs(data, errorCode); 112fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(!encodeUniqueCEs(errorCode)) { return FALSE; } 113fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(shortPrimaryOverflow) { 114fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Give digits long mini primaries, 115fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // so that there are more short primaries for letters. 116fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius firstShortPrimary = firstLatinPrimary; 117fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius resetCEs(); 118fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius getCEs(data, errorCode); 119fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(!encodeUniqueCEs(errorCode)) { return FALSE; } 120fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 121fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Note: If we still have a short-primary overflow but not a long-primary overflow, 122fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // then we could calculate how many more long primaries would fit, 123fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // and set the firstShortPrimary to that many after the current firstShortPrimary, 124fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // and try again. 125fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // However, this might only benefit the en_US_POSIX tailoring, 126fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // and it is simpler to suppress building fast Latin data for it in genrb, 127fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // or by returning FALSE here if shortPrimaryOverflow. 128fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 129fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UBool ok = !shortPrimaryOverflow && 130fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius encodeCharCEs(errorCode) && encodeContractions(errorCode); 131fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius contractionCEs.removeAllElements(); // might reduce heap memory usage 132fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uniqueCEs.removeAllElements(); 133fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return ok; 134fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 135fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 136fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusUBool 137fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::loadGroups(const CollationData &data, UErrorCode &errorCode) { 138fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_FAILURE(errorCode)) { return FALSE; } 139fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.append(0); // reserved for version & headerLength 140fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // The first few reordering groups should be special groups 141fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // (space, punct, ..., digit) followed by Latn, then Grek and other scripts. 142fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(int32_t i = 0;;) { 143fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(i >= data.scriptsLength) { 144fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // no Latn script 145fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius errorCode = U_INTERNAL_PROGRAM_ERROR; 146fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 147fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 148fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t head = data.scripts[i]; 149fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t lastByte = head & 0xff; // last primary byte in the group 150fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t group = data.scripts[i + 2]; 151fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(group == UCOL_REORDER_CODE_DIGIT) { 152fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius firstDigitPrimary = (head & 0xff00) << 16; 153fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius headerLength = result.length(); 154fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t r0 = (CollationFastLatin::VERSION << 8) | headerLength; 155fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.setCharAt(0, (UChar)r0); 156fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(group == USCRIPT_LATIN) { 157fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(firstDigitPrimary == 0) { 158fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // no digit group 159fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius errorCode = U_INTERNAL_PROGRAM_ERROR; 160fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 161fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 162fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius firstLatinPrimary = (head & 0xff00) << 16; 163fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius lastLatinPrimary = (lastByte << 24) | 0xffffff; 164fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 165fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(firstDigitPrimary == 0) { 166fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // a group below digits 167fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(lastByte > 0x7f) { 168fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // We only use 7 bits for the last byte of a below-digits group. 169fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // This does not warrant an errorCode, but we do not build a fast Latin table. 170fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 171fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 172fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.append((UChar)lastByte); 173fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 174fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius i = i + 2 + data.scripts[i + 1]; 175fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 176fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return TRUE; 177fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 178fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 179fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusUBool 180fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::inSameGroup(uint32_t p, uint32_t q) const { 181fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Both or neither need to be encoded as short primaries, 182fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // so that we can test only one and use the same bit mask. 183fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(p >= firstShortPrimary) { 184fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return q >= firstShortPrimary; 185fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(q >= firstShortPrimary) { 186fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 187fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 188fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Both or neither must be potentially-variable, 189fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // so that we can test only one and determine if both are variable. 190fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(p >= firstDigitPrimary) { 191fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return q >= firstDigitPrimary; 192fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(q >= firstDigitPrimary) { 193fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 194fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 195fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Both will be encoded with long mini primaries. 196fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // They must be in the same special reordering group, 197fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // so that we can test only one and determine if both are variable. 198fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius p >>= 24; // first primary byte 199fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius q >>= 24; 200fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(p != 0 && q != 0); 201fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(p <= result[headerLength - 1]); // the loop will terminate 202fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(int32_t i = 1;; ++i) { 203fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t lastByte = result[i]; 204fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(p <= lastByte) { 205fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return q <= lastByte; 206fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(q <= lastByte) { 207fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 208fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 209fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 210fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 211fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 212fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusvoid 213fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::resetCEs() { 214fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius contractionCEs.removeAllElements(); 215fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uniqueCEs.removeAllElements(); 216fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius shortPrimaryOverflow = FALSE; 217fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.truncate(headerLength); 218fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 219fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 220fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusvoid 221fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::getCEs(const CollationData &data, UErrorCode &errorCode) { 222fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_FAILURE(errorCode)) { return; } 223fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t i = 0; 224fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(UChar c = 0;; ++i, ++c) { 225fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(c == CollationFastLatin::LATIN_LIMIT) { 226fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius c = CollationFastLatin::PUNCT_START; 227fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(c == CollationFastLatin::PUNCT_LIMIT) { 228fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 229fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 230fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius const CollationData *d; 231fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t ce32 = data.getCE32(c); 232fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ce32 == Collation::FALLBACK_CE32) { 233fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius d = data.base; 234fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = d->getCE32(c); 235fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 236fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius d = &data; 237fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 238fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(getCEsFromCE32(*d, c, ce32, errorCode)) { 239fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius charCEs[i][0] = ce0; 240fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius charCEs[i][1] = ce1; 241fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addUniqueCE(ce0, errorCode); 242fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addUniqueCE(ce1, errorCode); 243fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 244fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // bail out for c 245fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius charCEs[i][0] = ce0 = Collation::NO_CE; 246fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius charCEs[i][1] = ce1 = 0; 247fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 248fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(c == 0 && !isContractionCharCE(ce0)) { 249fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Always map U+0000 to a contraction. 250fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Write a contraction list with only a default value if there is no real contraction. 251fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(contractionCEs.isEmpty()); 252fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addContractionEntry(CollationFastLatin::CONTR_CHAR_MASK, ce0, ce1, errorCode); 253fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius charCEs[0][0] = ((int64_t)Collation::NO_CE_PRIMARY << 32) | CONTRACTION_FLAG; 254fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius charCEs[0][1] = 0; 255fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 256fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 257fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Terminate the last contraction list. 258fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius contractionCEs.addElement(CollationFastLatin::CONTR_CHAR_MASK, errorCode); 259fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 260fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 261fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusUBool 262fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::getCEsFromCE32(const CollationData &data, UChar32 c, uint32_t ce32, 263fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UErrorCode &errorCode) { 264fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_FAILURE(errorCode)) { return FALSE; } 265fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = data.getFinalCE32(ce32); 266fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce1 = 0; 267fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(Collation::isSimpleOrLongCE32(ce32)) { 268fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce0 = Collation::ceFromCE32(ce32); 269fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 270fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius switch(Collation::tagFromCE32(ce32)) { 271fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::LATIN_EXPANSION_TAG: 272fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce0 = Collation::latinCE0FromCE32(ce32); 273fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce1 = Collation::latinCE1FromCE32(ce32); 274fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 275fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::EXPANSION32_TAG: { 276fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius const uint32_t *ce32s = data.ce32s + Collation::indexFromCE32(ce32); 277fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t length = Collation::lengthFromCE32(ce32); 278fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(length <= 2) { 279fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce0 = Collation::ceFromCE32(ce32s[0]); 280fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(length == 2) { 281fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce1 = Collation::ceFromCE32(ce32s[1]); 282fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 283fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 284fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 285fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 286fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 287fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 288fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::EXPANSION_TAG: { 289fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius const int64_t *ces = data.ces + Collation::indexFromCE32(ce32); 290fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t length = Collation::lengthFromCE32(ce32); 291fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(length <= 2) { 292fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce0 = ces[0]; 293fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(length == 2) { 294fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce1 = ces[1]; 295fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 296fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 297fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 298fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 299fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 300fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 301fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Note: We could support PREFIX_TAG (assert c>=0) 302fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // by recursing on its default CE32 and checking that none of the prefixes starts 303fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // with a fast Latin character. 304fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // However, currently (2013) there are only the L-before-middle-dot 305fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // prefix mappings in the Latin range, and those would be rejected anyway. 306fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::CONTRACTION_TAG: 307fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(c >= 0); 308fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return getCEsFromContractionCE32(data, ce32, errorCode); 309fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::OFFSET_TAG: 310fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(c >= 0); 311fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce0 = data.getCEFromOffsetCE32(c, ce32); 312fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 313fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius default: 314fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 315fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 316fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 317fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // A mapping can be completely ignorable. 318fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ce0 == 0) { return ce1 == 0; } 319fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // We do not support an ignorable ce0 unless it is completely ignorable. 320fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t p0 = (uint32_t)(ce0 >> 32); 321fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(p0 == 0) { return FALSE; } 322fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // We only support primaries up to the Latin script. 323fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(p0 > lastLatinPrimary) { return FALSE; } 324fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // We support non-common secondary and case weights only together with short primaries. 325fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t lower32_0 = (uint32_t)ce0; 326fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(p0 < firstShortPrimary) { 327fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t sc0 = lower32_0 & Collation::SECONDARY_AND_CASE_MASK; 328fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(sc0 != Collation::COMMON_SECONDARY_CE) { return FALSE; } 329fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 330fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // No below-common tertiary weights. 331fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if((lower32_0 & Collation::ONLY_TERTIARY_MASK) < Collation::COMMON_WEIGHT16) { return FALSE; } 332fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ce1 != 0) { 333fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Both primaries must be in the same group, 334fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // or both must get short mini primaries, 335fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // or a short-primary CE is followed by a secondary CE. 336fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // This is so that we can test the first primary and use the same mask for both, 337fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // and determine for both whether they are variable. 338fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t p1 = (uint32_t)(ce1 >> 32); 339fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(p1 == 0 ? p0 < firstShortPrimary : !inSameGroup(p0, p1)) { return FALSE; } 340fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t lower32_1 = (uint32_t)ce1; 341fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // No tertiary CEs. 342fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if((lower32_1 >> 16) == 0) { return FALSE; } 343fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // We support non-common secondary and case weights 344fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // only for secondary CEs or together with short primaries. 345fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(p1 != 0 && p1 < firstShortPrimary) { 346fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t sc1 = lower32_1 & Collation::SECONDARY_AND_CASE_MASK; 347fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(sc1 != Collation::COMMON_SECONDARY_CE) { return FALSE; } 348fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 349fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // No below-common tertiary weights. 350fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if((lower32_1 & Collation::ONLY_TERTIARY_MASK) < Collation::COMMON_WEIGHT16) { return FALSE; } 351fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 352fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // No quaternary weights. 353fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(((ce0 | ce1) & Collation::QUATERNARY_MASK) != 0) { return FALSE; } 354fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return TRUE; 355fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 356fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 357fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusUBool 358fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::getCEsFromContractionCE32(const CollationData &data, uint32_t ce32, 359fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UErrorCode &errorCode) { 360fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_FAILURE(errorCode)) { return FALSE; } 361fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius const UChar *p = data.contexts + Collation::indexFromCE32(ce32); 362fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = CollationData::readCE32(p); // Default if no suffix match. 363fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Since the original ce32 is not a prefix mapping, 364fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // the default ce32 must not be another contraction. 365fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(!Collation::isContractionCE32(ce32)); 366fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t contractionIndex = contractionCEs.size(); 367fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(getCEsFromCE32(data, U_SENTINEL, ce32, errorCode)) { 368fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addContractionEntry(CollationFastLatin::CONTR_CHAR_MASK, ce0, ce1, errorCode); 369fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 370fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Bail out for c-without-contraction. 371fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addContractionEntry(CollationFastLatin::CONTR_CHAR_MASK, Collation::NO_CE, 0, errorCode); 372fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 373fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Handle an encodable contraction unless the next contraction is too long 374fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // and starts with the same character. 375fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t prevX = -1; 376fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UBool addContraction = FALSE; 377fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UCharsTrie::Iterator suffixes(p + 2, 0, errorCode); 378fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius while(suffixes.next(errorCode)) { 379fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius const UnicodeString &suffix = suffixes.getString(); 380fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t x = CollationFastLatin::getCharIndex(suffix.charAt(0)); 381fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(x < 0) { continue; } // ignore anything but fast Latin text 382fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(x == prevX) { 383fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(addContraction) { 384fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Bail out for all contractions starting with this character. 385fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addContractionEntry(x, Collation::NO_CE, 0, errorCode); 386fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addContraction = FALSE; 387fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 388fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius continue; 389fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 390fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(addContraction) { 391fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addContractionEntry(prevX, ce0, ce1, errorCode); 392fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 393fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = (uint32_t)suffixes.getValue(); 394fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(suffix.length() == 1 && getCEsFromCE32(data, U_SENTINEL, ce32, errorCode)) { 395fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addContraction = TRUE; 396fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 397fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addContractionEntry(x, Collation::NO_CE, 0, errorCode); 398fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addContraction = FALSE; 399fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 400fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius prevX = x; 401fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 402fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(addContraction) { 403fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addContractionEntry(prevX, ce0, ce1, errorCode); 404fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 405fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_FAILURE(errorCode)) { return FALSE; } 406fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Note: There might not be any fast Latin contractions, but 407fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // we need to enter contraction handling anyway so that we can bail out 408fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // when there is a non-fast-Latin character following. 409fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // For example: Danish &Y<<u+umlaut, when we compare Y vs. u\u0308 we need to see the 410fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // following umlaut and bail out, rather than return the difference of Y vs. u. 411fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce0 = ((int64_t)Collation::NO_CE_PRIMARY << 32) | CONTRACTION_FLAG | contractionIndex; 412fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce1 = 0; 413fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return TRUE; 414fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 415fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 416fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusvoid 417fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::addContractionEntry(int32_t x, int64_t cce0, int64_t cce1, 418fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UErrorCode &errorCode) { 419fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius contractionCEs.addElement(x, errorCode); 420fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius contractionCEs.addElement(cce0, errorCode); 421fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius contractionCEs.addElement(cce1, errorCode); 422fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addUniqueCE(cce0, errorCode); 423fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius addUniqueCE(cce1, errorCode); 424fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 425fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 426fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusvoid 427fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::addUniqueCE(int64_t ce, UErrorCode &errorCode) { 428fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_FAILURE(errorCode)) { return; } 429fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ce == 0 || (uint32_t)(ce >> 32) == Collation::NO_CE_PRIMARY) { return; } 430fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce &= ~(int64_t)Collation::CASE_MASK; // blank out case bits 431fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t i = binarySearch(uniqueCEs.getBuffer(), uniqueCEs.size(), ce); 432fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(i < 0) { 433fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uniqueCEs.insertElementAt(ce, ~i, errorCode); 434fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 435fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 436fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 437fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusuint32_t 438fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::getMiniCE(int64_t ce) const { 439fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce &= ~(int64_t)Collation::CASE_MASK; // blank out case bits 440fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t index = binarySearch(uniqueCEs.getBuffer(), uniqueCEs.size(), ce); 441fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(index >= 0); 442fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return miniCEs[index]; 443fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 444fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 445fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusUBool 446fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::encodeUniqueCEs(UErrorCode &errorCode) { 447fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_FAILURE(errorCode)) { return FALSE; } 448fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uprv_free(miniCEs); 449fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCEs = (uint16_t *)uprv_malloc(uniqueCEs.size() * 2); 450fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(miniCEs == NULL) { 451fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius errorCode = U_MEMORY_ALLOCATION_ERROR; 452fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 453fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 454fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t group = 1; 455fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t lastGroupByte = result[group]; 456fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // The lowest unique CE must be at least a secondary CE. 457fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(((uint32_t)uniqueCEs.elementAti(0) >> 16) != 0); 458fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t prevPrimary = 0; 459fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t prevSecondary = 0; 460fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t pri = 0; 461fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t sec = 0; 462fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t ter = CollationFastLatin::COMMON_TER; 463fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(int32_t i = 0; i < uniqueCEs.size(); ++i) { 464fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int64_t ce = uniqueCEs.elementAti(i); 465fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Note: At least one of the p/s/t weights changes from one unique CE to the next. 466fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // (uniqueCEs does not store case bits.) 467fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t p = (uint32_t)(ce >> 32); 468fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(p != prevPrimary) { 469fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t p1 = p >> 24; 470fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius while(p1 > lastGroupByte) { 471fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(pri <= CollationFastLatin::MAX_LONG); 472fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Add the last "long primary" in or before the group 473fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // into the upper 9 bits of the group entry. 474fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.setCharAt(group, (UChar)((pri << 4) | lastGroupByte)); 475fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(++group < headerLength) { // group is 1-based 476fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius lastGroupByte = result[group]; 477fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 478fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius lastGroupByte = 0xff; 479fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 480fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 481fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 482fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(p < firstShortPrimary) { 483fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(pri == 0) { 484fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius pri = CollationFastLatin::MIN_LONG; 485fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(pri < CollationFastLatin::MAX_LONG) { 486fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius pri += CollationFastLatin::LONG_INC; 487fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 488fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#if DEBUG_COLLATION_FAST_LATIN_BUILDER 489fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius printf("long-primary overflow for %08x\n", p); 490fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#endif 491fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCEs[i] = CollationFastLatin::BAIL_OUT; 492fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius continue; 493fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 494fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 495fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(pri < CollationFastLatin::MIN_SHORT) { 496fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius pri = CollationFastLatin::MIN_SHORT; 497fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(pri < (CollationFastLatin::MAX_SHORT - CollationFastLatin::SHORT_INC)) { 498fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Reserve the highest primary weight for U+FFFF. 499fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius pri += CollationFastLatin::SHORT_INC; 500fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 501fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#if DEBUG_COLLATION_FAST_LATIN_BUILDER 502fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius printf("short-primary overflow for %08x\n", p); 503fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#endif 504fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius shortPrimaryOverflow = TRUE; 505fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCEs[i] = CollationFastLatin::BAIL_OUT; 506fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius continue; 507fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 508fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 509fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius prevPrimary = p; 510fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius prevSecondary = Collation::COMMON_WEIGHT16; 511fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius sec = CollationFastLatin::COMMON_SEC; 512fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ter = CollationFastLatin::COMMON_TER; 513fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 514fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t lower32 = (uint32_t)ce; 515fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t s = lower32 >> 16; 516fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(s != prevSecondary) { 517fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(pri == 0) { 518fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(sec == 0) { 519fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius sec = CollationFastLatin::MIN_SEC_HIGH; 520fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(sec < CollationFastLatin::MAX_SEC_HIGH) { 521fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius sec += CollationFastLatin::SEC_INC; 522fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 523fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCEs[i] = CollationFastLatin::BAIL_OUT; 524fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius continue; 525fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 526fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius prevSecondary = s; 527fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ter = CollationFastLatin::COMMON_TER; 528fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(s < Collation::COMMON_WEIGHT16) { 529fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(sec == CollationFastLatin::COMMON_SEC) { 530fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius sec = CollationFastLatin::MIN_SEC_BEFORE; 531fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(sec < CollationFastLatin::MAX_SEC_BEFORE) { 532fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius sec += CollationFastLatin::SEC_INC; 533fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 534fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCEs[i] = CollationFastLatin::BAIL_OUT; 535fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius continue; 536fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 537fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(s == Collation::COMMON_WEIGHT16) { 538fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius sec = CollationFastLatin::COMMON_SEC; 539fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 540fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(sec < CollationFastLatin::MIN_SEC_AFTER) { 541fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius sec = CollationFastLatin::MIN_SEC_AFTER; 542fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(sec < CollationFastLatin::MAX_SEC_AFTER) { 543fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius sec += CollationFastLatin::SEC_INC; 544fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 545fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCEs[i] = CollationFastLatin::BAIL_OUT; 546fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius continue; 547fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 548fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 549fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius prevSecondary = s; 550fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ter = CollationFastLatin::COMMON_TER; 551fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 552fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT((lower32 & Collation::CASE_MASK) == 0); // blanked out in uniqueCEs 553fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t t = lower32 & Collation::ONLY_TERTIARY_MASK; 554fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(t > Collation::COMMON_WEIGHT16) { 555fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ter < CollationFastLatin::MAX_TER_AFTER) { 556fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ++ter; 557fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 558fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCEs[i] = CollationFastLatin::BAIL_OUT; 559fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius continue; 560fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 561fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 562fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(CollationFastLatin::MIN_LONG <= pri && pri <= CollationFastLatin::MAX_LONG) { 563fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(sec == CollationFastLatin::COMMON_SEC); 564fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCEs[i] = (uint16_t)(pri | ter); 565fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 566fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCEs[i] = (uint16_t)(pri | sec | ter); 567fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 568fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 569fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#if DEBUG_COLLATION_FAST_LATIN_BUILDER 570fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius printf("last mini primary: %04x\n", pri); 571fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#endif 572fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#if DEBUG_COLLATION_FAST_LATIN_BUILDER >= 2 573fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(int32_t i = 0; i < uniqueCEs.size(); ++i) { 574fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int64_t ce = uniqueCEs.elementAti(i); 575fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius printf("unique CE 0x%016lx -> 0x%04x\n", ce, miniCEs[i]); 576fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 577fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#endif 578fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return U_SUCCESS(errorCode); 579fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 580fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 581fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusUBool 582fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::encodeCharCEs(UErrorCode &errorCode) { 583fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_FAILURE(errorCode)) { return FALSE; } 584fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t miniCEsStart = result.length(); 585fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(int32_t i = 0; i < CollationFastLatin::NUM_FAST_CHARS; ++i) { 586fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.append(0); // initialize to completely ignorable 587fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 588fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t indexBase = result.length(); 589fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(int32_t i = 0; i < CollationFastLatin::NUM_FAST_CHARS; ++i) { 590fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int64_t ce = charCEs[i][0]; 591fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(isContractionCharCE(ce)) { continue; } // defer contraction 592fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t miniCE = encodeTwoCEs(ce, charCEs[i][1]); 593fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(miniCE > 0xffff) { 594fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Note: There is a chance that this new expansion is the same as a previous one, 595fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // and if so, then we could reuse the other expansion. 596fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // However, that seems unlikely. 597fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t expansionIndex = result.length() - indexBase; 598fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(expansionIndex > (int32_t)CollationFastLatin::INDEX_MASK) { 599fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCE = CollationFastLatin::BAIL_OUT; 600fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 601fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.append((UChar)(miniCE >> 16)).append((UChar)miniCE); 602fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCE = CollationFastLatin::EXPANSION | expansionIndex; 603fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 604fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 605fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.setCharAt(miniCEsStart + i, (UChar)miniCE); 606fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 607fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return U_SUCCESS(errorCode); 608fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 609fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 610fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusUBool 611fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::encodeContractions(UErrorCode &errorCode) { 612fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // We encode all contraction lists so that the first word of a list 613fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // terminates the previous list, and we only need one additional terminator at the end. 614fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_FAILURE(errorCode)) { return FALSE; } 615fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t indexBase = headerLength + CollationFastLatin::NUM_FAST_CHARS; 616fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t firstContractionIndex = result.length(); 617fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(int32_t i = 0; i < CollationFastLatin::NUM_FAST_CHARS; ++i) { 618fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int64_t ce = charCEs[i][0]; 619fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(!isContractionCharCE(ce)) { continue; } 620fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t contractionIndex = result.length() - indexBase; 621fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(contractionIndex > (int32_t)CollationFastLatin::INDEX_MASK) { 622fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.setCharAt(headerLength + i, CollationFastLatin::BAIL_OUT); 623fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius continue; 624fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 625fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UBool firstTriple = TRUE; 626fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(int32_t index = (int32_t)ce & 0x7fffffff;; index += 3) { 627fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t x = contractionCEs.elementAti(index); 628fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if((uint32_t)x == CollationFastLatin::CONTR_CHAR_MASK && !firstTriple) { break; } 629fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int64_t cce0 = contractionCEs.elementAti(index + 1); 630fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int64_t cce1 = contractionCEs.elementAti(index + 2); 631fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t miniCE = encodeTwoCEs(cce0, cce1); 632fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(miniCE == CollationFastLatin::BAIL_OUT) { 633fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.append((UChar)(x | (1 << CollationFastLatin::CONTR_LENGTH_SHIFT))); 634fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(miniCE <= 0xffff) { 635fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.append((UChar)(x | (2 << CollationFastLatin::CONTR_LENGTH_SHIFT))); 636fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.append((UChar)miniCE); 637fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 638fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.append((UChar)(x | (3 << CollationFastLatin::CONTR_LENGTH_SHIFT))); 639fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.append((UChar)(miniCE >> 16)).append((UChar)miniCE); 640fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 641fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius firstTriple = FALSE; 642fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 643fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Note: There is a chance that this new contraction list is the same as a previous one, 644fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // and if so, then we could truncate the result and reuse the other list. 645fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // However, that seems unlikely. 646fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.setCharAt(headerLength + i, 647fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius (UChar)(CollationFastLatin::CONTRACTION | contractionIndex)); 648fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 649fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(result.length() > firstContractionIndex) { 650fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Terminate the last contraction list. 651fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius result.append((UChar)CollationFastLatin::CONTR_CHAR_MASK); 652fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 653fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(result.isBogus()) { 654fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius errorCode = U_MEMORY_ALLOCATION_ERROR; 655fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 656fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 657fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#if DEBUG_COLLATION_FAST_LATIN_BUILDER 658fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius printf("** fast Latin %d * 2 = %d bytes\n", result.length(), result.length() * 2); 659fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius puts(" header & below-digit groups map"); 660fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t i = 0; 661fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(; i < headerLength; ++i) { 662fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius printf(" %04x", result[i]); 663fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 664fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius printf("\n char mini CEs"); 665fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(CollationFastLatin::NUM_FAST_CHARS % 16 == 0); 666fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(; i < indexBase; i += 16) { 667fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UChar32 c = i - headerLength; 668fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(c >= CollationFastLatin::LATIN_LIMIT) { 669fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius c = CollationFastLatin::PUNCT_START + c - CollationFastLatin::LATIN_LIMIT; 670fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 671fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius printf("\n %04x:", c); 672fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(int32_t j = 0; j < 16; ++j) { 673fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius printf(" %04x", result[i + j]); 674fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 675fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 676fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius printf("\n expansions & contractions"); 677fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(; i < result.length(); ++i) { 678fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if((i - indexBase) % 16 == 0) { puts(""); } 679fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius printf(" %04x", result[i]); 680fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 681fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius puts(""); 682fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#endif 683fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return TRUE; 684fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 685fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 686fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusuint32_t 687fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationFastLatinBuilder::encodeTwoCEs(int64_t first, int64_t second) const { 688fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(first == 0) { 689fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return 0; // completely ignorable 690fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 691fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(first == Collation::NO_CE) { 692fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return CollationFastLatin::BAIL_OUT; 693fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 694fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT((uint32_t)(first >> 32) != Collation::NO_CE_PRIMARY); 695fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 696fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t miniCE = getMiniCE(first); 697fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(miniCE == CollationFastLatin::BAIL_OUT) { return miniCE; } 698fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(miniCE >= CollationFastLatin::MIN_SHORT) { 699fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Extract & copy the case bits. 700fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Shift them from normal CE bits 15..14 to mini CE bits 4..3. 701fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t c = (((uint32_t)first & Collation::CASE_MASK) >> (14 - 3)); 702fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Only in mini CEs: Ignorable case bits = 0, lowercase = 1. 703fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius c += CollationFastLatin::LOWER_CASE; 704fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCE |= c; 705fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 706fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(second == 0) { return miniCE; } 707fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 708fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t miniCE1 = getMiniCE(second); 709fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(miniCE1 == CollationFastLatin::BAIL_OUT) { return miniCE1; } 710fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 711fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t case1 = (uint32_t)second & Collation::CASE_MASK; 712fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(miniCE >= CollationFastLatin::MIN_SHORT && 713fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius (miniCE & CollationFastLatin::SECONDARY_MASK) == CollationFastLatin::COMMON_SEC) { 714fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Try to combine the two mini CEs into one. 715fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t sec1 = miniCE1 & CollationFastLatin::SECONDARY_MASK; 716fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t ter1 = miniCE1 & CollationFastLatin::TERTIARY_MASK; 717fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(sec1 >= CollationFastLatin::MIN_SEC_HIGH && case1 == 0 && 718fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ter1 == CollationFastLatin::COMMON_TER) { 719fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // sec1>=sec_high implies pri1==0. 720fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return (miniCE & ~CollationFastLatin::SECONDARY_MASK) | sec1; 721fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 722fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 723fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 724fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(miniCE1 <= CollationFastLatin::SECONDARY_MASK || CollationFastLatin::MIN_SHORT <= miniCE1) { 725fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Secondary CE, or a CE with a short primary, copy the case bits. 726fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case1 = (case1 >> (14 - 3)) + CollationFastLatin::LOWER_CASE; 727fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius miniCE1 |= case1; 728fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 729fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return (miniCE << 16) | miniCE1; 730fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 731fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 732fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusU_NAMESPACE_END 733fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 734fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#endif // !UCONFIG_NO_COLLATION 735