1fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius/* 2fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius******************************************************************************* 3fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius* Copyright (C) 2010-2014, International Business Machines 4fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius* Corporation and others. All Rights Reserved. 5fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius******************************************************************************* 6fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius* collationiterator.cpp 7fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius* 8fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius* created on: 2010oct27 9fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius* created by: Markus W. Scherer 10fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius*/ 11fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 12fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "utypeinfo.h" // for 'typeid' to work 13fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 14fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "unicode/utypes.h" 15fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 16fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#if !UCONFIG_NO_COLLATION 17fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 18fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "unicode/ucharstrie.h" 19fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "unicode/ustringtrie.h" 20fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "charstr.h" 21fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "cmemory.h" 22fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "collation.h" 23fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "collationdata.h" 24fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "collationfcd.h" 25fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "collationiterator.h" 26fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "normalizer2impl.h" 27fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "uassert.h" 28fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#include "uvectr32.h" 29fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 30fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusU_NAMESPACE_BEGIN 31fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 32fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationIterator::CEBuffer::~CEBuffer() {} 33fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 34fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusUBool 35fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationIterator::CEBuffer::ensureAppendCapacity(int32_t appCap, UErrorCode &errorCode) { 36fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t capacity = buffer.getCapacity(); 37fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if((length + appCap) <= capacity) { return TRUE; } 38fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_FAILURE(errorCode)) { return FALSE; } 39fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius do { 40fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(capacity < 1000) { 41fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius capacity *= 4; 42fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 43fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius capacity *= 2; 44fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 45fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } while(capacity < (length + appCap)); 46fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int64_t *p = buffer.resize(capacity, length); 47fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(p == NULL) { 48fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius errorCode = U_MEMORY_ALLOCATION_ERROR; 49fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 50fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 51fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return TRUE; 52fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 53fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 54fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius// State of combining marks skipped in discontiguous contraction. 55fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius// We create a state object on first use and keep it around deactivated between uses. 56fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusclass SkippedState : public UMemory { 57fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliuspublic: 58fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Born active but empty. 59fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius SkippedState() : pos(0), skipLengthAtMatch(0) {} 60fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius void clear() { 61fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius oldBuffer.remove(); 62fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius pos = 0; 63fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // The newBuffer is reset by setFirstSkipped(). 64fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 65fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 66fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UBool isEmpty() const { return oldBuffer.isEmpty(); } 67fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 68fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UBool hasNext() const { return pos < oldBuffer.length(); } 69fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 70fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Requires hasNext(). 71fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UChar32 next() { 72fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UChar32 c = oldBuffer.char32At(pos); 73fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius pos += U16_LENGTH(c); 74fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return c; 75fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 76fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 77fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Accounts for one more input code point read beyond the end of the marks buffer. 78fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius void incBeyond() { 79fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(!hasNext()); 80fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ++pos; 81fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 82fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 83fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Goes backward through the skipped-marks buffer. 84fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Returns the number of code points read beyond the skipped marks 85fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // that need to be backtracked through normal input. 86fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t backwardNumCodePoints(int32_t n) { 87fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t length = oldBuffer.length(); 88fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t beyond = pos - length; 89fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(beyond > 0) { 90fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(beyond >= n) { 91fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Not back far enough to re-enter the oldBuffer. 92fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius pos -= n; 93fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return n; 94fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 95fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Back out all beyond-oldBuffer code points and re-enter the buffer. 96fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius pos = oldBuffer.moveIndex32(length, beyond - n); 97fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return beyond; 98fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 99fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 100fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Go backwards from inside the oldBuffer. 101fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius pos = oldBuffer.moveIndex32(pos, -n); 102fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return 0; 103fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 104fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 105fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 106fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius void setFirstSkipped(UChar32 c) { 107fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius skipLengthAtMatch = 0; 108fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius newBuffer.setTo(c); 109fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 110fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 111fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius void skip(UChar32 c) { 112fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius newBuffer.append(c); 113fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 114fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 115fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius void recordMatch() { skipLengthAtMatch = newBuffer.length(); } 116fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 117fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Replaces the characters we consumed with the newly skipped ones. 118fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius void replaceMatch() { 119fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Note: UnicodeString.replace() pins pos to at most length(). 120fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius oldBuffer.replace(0, pos, newBuffer, 0, skipLengthAtMatch); 121fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius pos = 0; 122fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 123fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 124fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius void saveTrieState(const UCharsTrie &trie) { trie.saveState(state); } 125fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius void resetToTrieState(UCharsTrie &trie) const { trie.resetToState(state); } 126fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 127fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusprivate: 128fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Combining marks skipped in previous discontiguous-contraction matching. 129fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // After that discontiguous contraction was completed, we start reading them from here. 130fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UnicodeString oldBuffer; 131fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Combining marks newly skipped in current discontiguous-contraction matching. 132fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // These might have been read from the normal text or from the oldBuffer. 133fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UnicodeString newBuffer; 134fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Reading index in oldBuffer, 135fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // or counter for how many code points have been read beyond oldBuffer (pos-oldBuffer.length()). 136fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t pos; 137fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // newBuffer.length() at the time of the last matching character. 138fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // When a partial match fails, we back out skipped and partial-matching input characters. 139fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t skipLengthAtMatch; 140fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // We save the trie state before we attempt to match a character, 141fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // so that we can skip it and try the next one. 142fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UCharsTrie::State state; 143fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius}; 144fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 145fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationIterator::CollationIterator(const CollationIterator &other) 146fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius : UObject(other), 147fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius trie(other.trie), 148fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius data(other.data), 149fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius cesIndex(other.cesIndex), 150fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius skipped(NULL), 151fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius numCpFwd(other.numCpFwd), 152fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius isNumeric(other.isNumeric) { 153fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UErrorCode errorCode = U_ZERO_ERROR; 154fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t length = other.ceBuffer.length; 155fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(length > 0 && ceBuffer.ensureAppendCapacity(length, errorCode)) { 156fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(int32_t i = 0; i < length; ++i) { 157fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ceBuffer.set(i, other.ceBuffer.get(i)); 158fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 159fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ceBuffer.length = length; 160fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 161fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius cesIndex = 0; 162fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 163fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 164fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 165fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationIterator::~CollationIterator() { 166fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius delete skipped; 167fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 168fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 169fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusUBool 170fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationIterator::operator==(const CollationIterator &other) const { 171fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Subclasses: Call this method and then add more specific checks. 172fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Compare the iterator state but not the collation data (trie & data fields): 173fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Assume that the caller compares the data. 174fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Ignore skipped since that should be unused between calls to nextCE(). 175fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // (It only stays around to avoid another memory allocation.) 176fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(!(typeid(*this) == typeid(other) && 177fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ceBuffer.length == other.ceBuffer.length && 178fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius cesIndex == other.cesIndex && 179fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius numCpFwd == other.numCpFwd && 180fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius isNumeric == other.isNumeric)) { 181fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 182fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 183fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(int32_t i = 0; i < ceBuffer.length; ++i) { 184fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ceBuffer.get(i) != other.ceBuffer.get(i)) { return FALSE; } 185fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 186fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return TRUE; 187fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 188fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 189fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusvoid 190fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationIterator::reset() { 191fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius cesIndex = ceBuffer.length = 0; 192fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(skipped != NULL) { skipped->clear(); } 193fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 194fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 195fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusint32_t 196fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationIterator::fetchCEs(UErrorCode &errorCode) { 197fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius while(U_SUCCESS(errorCode) && nextCE(errorCode) != Collation::NO_CE) { 198fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // No need to loop for each expansion CE. 199fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius cesIndex = ceBuffer.length; 200fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 201fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return ceBuffer.length; 202fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 203fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 204fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusuint32_t 205fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationIterator::handleNextCE32(UChar32 &c, UErrorCode &errorCode) { 206fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius c = nextCodePoint(errorCode); 207fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return (c < 0) ? Collation::FALLBACK_CE32 : data->getCE32(c); 208fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 209fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 210fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusUChar 211fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationIterator::handleGetTrailSurrogate() { 212fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return 0; 213fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 214fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 215fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusUBool 216fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationIterator::foundNULTerminator() { 217fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 218fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 219fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 220fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusUBool 221fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationIterator::forbidSurrogateCodePoints() const { 222fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return FALSE; 223fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 224fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 225fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusuint32_t 226fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationIterator::getDataCE32(UChar32 c) const { 227fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return data->getCE32(c); 228fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 229fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 230fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusuint32_t 231fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationIterator::getCE32FromBuilderData(uint32_t /*ce32*/, UErrorCode &errorCode) { 232fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_SUCCESS(errorCode)) { errorCode = U_INTERNAL_PROGRAM_ERROR; } 233fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return 0; 234fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 235fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 236fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusint64_t 237fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationIterator::nextCEFromCE32(const CollationData *d, UChar32 c, uint32_t ce32, 238fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UErrorCode &errorCode) { 239fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius --ceBuffer.length; // Undo ceBuffer.incLength(). 240fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius appendCEsFromCE32(d, c, ce32, TRUE, errorCode); 241fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_SUCCESS(errorCode)) { 242fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return ceBuffer.get(cesIndex++); 243fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 244fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return Collation::NO_CE_PRIMARY; 245fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 246fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 247fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 248fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusvoid 249fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationIterator::appendCEsFromCE32(const CollationData *d, UChar32 c, uint32_t ce32, 250fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UBool forward, UErrorCode &errorCode) { 251fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius while(Collation::isSpecialCE32(ce32)) { 252fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius switch(Collation::tagFromCE32(ce32)) { 253fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::FALLBACK_TAG: 254fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::RESERVED_TAG_3: 255fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_SUCCESS(errorCode)) { errorCode = U_INTERNAL_PROGRAM_ERROR; } 256fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return; 257fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::LONG_PRIMARY_TAG: 258fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ceBuffer.append(Collation::ceFromLongPrimaryCE32(ce32), errorCode); 259fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return; 260fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::LONG_SECONDARY_TAG: 261fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ceBuffer.append(Collation::ceFromLongSecondaryCE32(ce32), errorCode); 262fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return; 263fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::LATIN_EXPANSION_TAG: 264fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ceBuffer.ensureAppendCapacity(2, errorCode)) { 265fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ceBuffer.set(ceBuffer.length, Collation::latinCE0FromCE32(ce32)); 266fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ceBuffer.set(ceBuffer.length + 1, Collation::latinCE1FromCE32(ce32)); 267fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ceBuffer.length += 2; 268fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 269fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return; 270fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::EXPANSION32_TAG: { 271fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius const uint32_t *ce32s = d->ce32s + Collation::indexFromCE32(ce32); 272fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t length = Collation::lengthFromCE32(ce32); 273fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ceBuffer.ensureAppendCapacity(length, errorCode)) { 274fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius do { 275fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ceBuffer.appendUnsafe(Collation::ceFromCE32(*ce32s++)); 276fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } while(--length > 0); 277fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 278fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return; 279fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 280fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::EXPANSION_TAG: { 281fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius const int64_t *ces = d->ces + Collation::indexFromCE32(ce32); 282fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t length = Collation::lengthFromCE32(ce32); 283fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ceBuffer.ensureAppendCapacity(length, errorCode)) { 284fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius do { 285fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ceBuffer.appendUnsafe(*ces++); 286fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } while(--length > 0); 287fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 288fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return; 289fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 290fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::BUILDER_DATA_TAG: 291fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = getCE32FromBuilderData(ce32, errorCode); 292fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_FAILURE(errorCode)) { return; } 293fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ce32 == Collation::FALLBACK_CE32) { 294fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius d = data->base; 295fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = d->getCE32(c); 296fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 297fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 298fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::PREFIX_TAG: 299fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(forward) { backwardNumCodePoints(1, errorCode); } 300fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = getCE32FromPrefix(d, ce32, errorCode); 301fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(forward) { forwardNumCodePoints(1, errorCode); } 302fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 303fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::CONTRACTION_TAG: { 304fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius const UChar *p = d->contexts + Collation::indexFromCE32(ce32); 305fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t defaultCE32 = CollationData::readCE32(p); // Default if no suffix match. 306fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(!forward) { 307fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Backward contractions are handled by previousCEUnsafe(). 308fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // c has contractions but they were not found. 309fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = defaultCE32; 310fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 311fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 312fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UChar32 nextCp; 313fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(skipped == NULL && numCpFwd < 0) { 314fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Some portion of nextCE32FromContraction() pulled out here as an ASCII fast path, 315fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // avoiding the function call and the nextSkippedCodePoint() overhead. 316fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius nextCp = nextCodePoint(errorCode); 317fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(nextCp < 0) { 318fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // No more text. 319fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = defaultCE32; 320fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 321fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if((ce32 & Collation::CONTRACT_NEXT_CCC) != 0 && 322fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius !CollationFCD::mayHaveLccc(nextCp)) { 323fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // All contraction suffixes start with characters with lccc!=0 324fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // but the next code point has lccc==0. 325fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius backwardNumCodePoints(1, errorCode); 326fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = defaultCE32; 327fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 328fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 329fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 330fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius nextCp = nextSkippedCodePoint(errorCode); 331fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(nextCp < 0) { 332fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // No more text. 333fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = defaultCE32; 334fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 335fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if((ce32 & Collation::CONTRACT_NEXT_CCC) != 0 && 336fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius !CollationFCD::mayHaveLccc(nextCp)) { 337fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // All contraction suffixes start with characters with lccc!=0 338fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // but the next code point has lccc==0. 339fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius backwardNumSkipped(1, errorCode); 340fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = defaultCE32; 341fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 342fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 343fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 344fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = nextCE32FromContraction(d, ce32, p + 2, defaultCE32, nextCp, errorCode); 345fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ce32 == Collation::NO_CE32) { 346fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // CEs from a discontiguous contraction plus the skipped combining marks 347fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // have been appended already. 348fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return; 349fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 350fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 351fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 352fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::DIGIT_TAG: 353fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(isNumeric) { 354fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius appendNumericCEs(ce32, forward, errorCode); 355fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return; 356fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 357fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Fetch the non-numeric-collation CE32 and continue. 358fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = d->ce32s[Collation::indexFromCE32(ce32)]; 359fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 360fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 361fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::U0000_TAG: 362fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(c == 0); 363fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(forward && foundNULTerminator()) { 364fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Handle NUL-termination. (Not needed in Java.) 365fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ceBuffer.append(Collation::NO_CE, errorCode); 366fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return; 367fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 368fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Fetch the normal ce32 for U+0000 and continue. 369fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = d->ce32s[0]; 370fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 371fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 372fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::HANGUL_TAG: { 373fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius const uint32_t *jamoCE32s = d->jamoCE32s; 374fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius c -= Hangul::HANGUL_BASE; 375fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UChar32 t = c % Hangul::JAMO_T_COUNT; 376fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius c /= Hangul::JAMO_T_COUNT; 377fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UChar32 v = c % Hangul::JAMO_V_COUNT; 378fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius c /= Hangul::JAMO_V_COUNT; 379fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if((ce32 & Collation::HANGUL_NO_SPECIAL_JAMO) != 0) { 380fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // None of the Jamo CE32s are isSpecialCE32(). 381fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Avoid recursive function calls and per-Jamo tests. 382fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ceBuffer.ensureAppendCapacity(t == 0 ? 2 : 3, errorCode)) { 383fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ceBuffer.set(ceBuffer.length, Collation::ceFromCE32(jamoCE32s[c])); 384fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ceBuffer.set(ceBuffer.length + 1, Collation::ceFromCE32(jamoCE32s[19 + v])); 385fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ceBuffer.length += 2; 386fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(t != 0) { 387fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ceBuffer.appendUnsafe(Collation::ceFromCE32(jamoCE32s[39 + t])); 388fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 389fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 390fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return; 391fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 392fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // We should not need to compute each Jamo code point. 393fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // In particular, there should be no offset or implicit ce32. 394fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius appendCEsFromCE32(d, U_SENTINEL, jamoCE32s[c], forward, errorCode); 395fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius appendCEsFromCE32(d, U_SENTINEL, jamoCE32s[19 + v], forward, errorCode); 396fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(t == 0) { return; } 397fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // offset 39 = 19 + 21 - 1: 398fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // 19 = JAMO_L_COUNT 399fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // 21 = JAMO_T_COUNT 400fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // -1 = omit t==0 401fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = jamoCE32s[39 + t]; 402fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius c = U_SENTINEL; 403fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 404fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 405fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 406fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::LEAD_SURROGATE_TAG: { 407fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(forward); // Backward iteration should never see lead surrogate code _unit_ data. 408fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(U16_IS_LEAD(c)); 409fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UChar trail; 410fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U16_IS_TRAIL(trail = handleGetTrailSurrogate())) { 411fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius c = U16_GET_SUPPLEMENTARY(c, trail); 412fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 &= Collation::LEAD_TYPE_MASK; 413fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ce32 == Collation::LEAD_ALL_UNASSIGNED) { 414fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = Collation::UNASSIGNED_CE32; // unassigned-implicit 415fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(ce32 == Collation::LEAD_ALL_FALLBACK || 416fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius (ce32 = d->getCE32FromSupplementary(c)) == Collation::FALLBACK_CE32) { 417fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // fall back to the base data 418fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius d = d->base; 419fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = d->getCE32FromSupplementary(c); 420fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 421fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 422fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // c is an unpaired surrogate. 423fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = Collation::UNASSIGNED_CE32; 424fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 425fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 426fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 427fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::OFFSET_TAG: 428fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(c >= 0); 429fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ceBuffer.append(d->getCEFromOffsetCE32(c, ce32), errorCode); 430fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return; 431fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius case Collation::IMPLICIT_TAG: 432fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(c >= 0); 433fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_IS_SURROGATE(c) && forbidSurrogateCodePoints()) { 434fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = Collation::FFFD_CE32; 435fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 436fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 437fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ceBuffer.append(Collation::unassignedCEFromCodePoint(c), errorCode); 438fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return; 439fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 440fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 441fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 442fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ceBuffer.append(Collation::ceFromSimpleCE32(ce32), errorCode); 443fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 444fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 445fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusuint32_t 446fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationIterator::getCE32FromPrefix(const CollationData *d, uint32_t ce32, 447fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UErrorCode &errorCode) { 448fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius const UChar *p = d->contexts + Collation::indexFromCE32(ce32); 449fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = CollationData::readCE32(p); // Default if no prefix match. 450fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius p += 2; 451fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Number of code points read before the original code point. 452fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t lookBehind = 0; 453fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UCharsTrie prefixes(p); 454fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(;;) { 455fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UChar32 c = previousCodePoint(errorCode); 456fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(c < 0) { break; } 457fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ++lookBehind; 458fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UStringTrieResult match = prefixes.nextForCodePoint(c); 459fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(USTRINGTRIE_HAS_VALUE(match)) { 460fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = (uint32_t)prefixes.getValue(); 461fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 462fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(!USTRINGTRIE_HAS_NEXT(match)) { break; } 463fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 464fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius forwardNumCodePoints(lookBehind, errorCode); 465fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return ce32; 466fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 467fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 468fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusUChar32 469fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationIterator::nextSkippedCodePoint(UErrorCode &errorCode) { 470fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(skipped != NULL && skipped->hasNext()) { return skipped->next(); } 471fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(numCpFwd == 0) { return U_SENTINEL; } 472fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UChar32 c = nextCodePoint(errorCode); 473fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(skipped != NULL && !skipped->isEmpty() && c >= 0) { skipped->incBeyond(); } 474fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(numCpFwd > 0 && c >= 0) { --numCpFwd; } 475fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return c; 476fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 477fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 478fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusvoid 479fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationIterator::backwardNumSkipped(int32_t n, UErrorCode &errorCode) { 480fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(skipped != NULL && !skipped->isEmpty()) { 481fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius n = skipped->backwardNumCodePoints(n); 482fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 483fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius backwardNumCodePoints(n, errorCode); 484fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(numCpFwd >= 0) { numCpFwd += n; } 485fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 486fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 487fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusuint32_t 488fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationIterator::nextCE32FromContraction(const CollationData *d, uint32_t contractionCE32, 489fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius const UChar *p, uint32_t ce32, UChar32 c, 490fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UErrorCode &errorCode) { 491fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // c: next code point after the original one 492fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 493fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Number of code points read beyond the original code point. 494fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Needed for discontiguous contraction matching. 495fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t lookAhead = 1; 496fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Number of code points read since the last match (initially only c). 497fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t sinceMatch = 1; 498fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Normally we only need a contiguous match, 499fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // and therefore need not remember the suffixes state from before a mismatch for retrying. 500fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // If we are already processing skipped combining marks, then we do track the state. 501fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UCharsTrie suffixes(p); 502fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(skipped != NULL && !skipped->isEmpty()) { skipped->saveTrieState(suffixes); } 503fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UStringTrieResult match = suffixes.firstForCodePoint(c); 504fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(;;) { 505fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UChar32 nextCp; 506fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(USTRINGTRIE_HAS_VALUE(match)) { 507fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = (uint32_t)suffixes.getValue(); 508fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(!USTRINGTRIE_HAS_NEXT(match) || (c = nextSkippedCodePoint(errorCode)) < 0) { 509fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return ce32; 510fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 511fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(skipped != NULL && !skipped->isEmpty()) { skipped->saveTrieState(suffixes); } 512fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius sinceMatch = 1; 513fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else if(match == USTRINGTRIE_NO_MATCH || (nextCp = nextSkippedCodePoint(errorCode)) < 0) { 514fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // No match for c, or partial match (USTRINGTRIE_NO_VALUE) and no further text. 515fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Back up if necessary, and try a discontiguous contraction. 516fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if((contractionCE32 & Collation::CONTRACT_TRAILING_CCC) != 0 && 517fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Discontiguous contraction matching extends an existing match. 518fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // If there is no match yet, then there is nothing to do. 519fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ((contractionCE32 & Collation::CONTRACT_SINGLE_CP_NO_MATCH) == 0 || 520fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius sinceMatch < lookAhead)) { 521fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // The last character of at least one suffix has lccc!=0, 522fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // allowing for discontiguous contractions. 523fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // UCA S2.1.1 only processes non-starters immediately following 524fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // "a match in the table" (sinceMatch=1). 525fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(sinceMatch > 1) { 526fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Return to the state after the last match. 527fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // (Return to sinceMatch=0 and re-fetch the first partially-matched character.) 528fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius backwardNumSkipped(sinceMatch, errorCode); 529fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius c = nextSkippedCodePoint(errorCode); 530fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius lookAhead -= sinceMatch - 1; 531fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius sinceMatch = 1; 532fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 533fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(d->getFCD16(c) > 0xff) { 534fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return nextCE32FromDiscontiguousContraction( 535fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius d, suffixes, ce32, lookAhead, c, errorCode); 536fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 537fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 538fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 539fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 540fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Continue after partial match (USTRINGTRIE_NO_VALUE) for c. 541fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // It does not have a result value, therefore it is not itself "a match in the table". 542fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // If a partially-matched c has ccc!=0 then 543fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // it might be skipped in discontiguous contraction. 544fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius c = nextCp; 545fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ++sinceMatch; 546fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 547fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ++lookAhead; 548fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius match = suffixes.nextForCodePoint(c); 549fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 550fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius backwardNumSkipped(sinceMatch, errorCode); 551fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return ce32; 552fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 553fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 554fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusuint32_t 555fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationIterator::nextCE32FromDiscontiguousContraction( 556fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius const CollationData *d, UCharsTrie &suffixes, uint32_t ce32, 557fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t lookAhead, UChar32 c, 558fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UErrorCode &errorCode) { 559fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_FAILURE(errorCode)) { return 0; } 560fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 561fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // UCA section 3.3.2 Contractions: 562fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Contractions that end with non-starter characters 563fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // are known as discontiguous contractions. 564fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // ... discontiguous contractions must be detected in input text 565fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // whenever the final sequence of non-starter characters could be rearranged 566fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // so as to make a contiguous matching sequence that is canonically equivalent. 567fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 568fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // UCA: http://www.unicode.org/reports/tr10/#S2.1 569fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // S2.1 Find the longest initial substring S at each point that has a match in the table. 570fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // S2.1.1 If there are any non-starters following S, process each non-starter C. 571fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // S2.1.2 If C is not blocked from S, find if S + C has a match in the table. 572fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Note: A non-starter in a string is called blocked 573fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // if there is another non-starter of the same canonical combining class or zero 574fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // between it and the last character of canonical combining class 0. 575fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // S2.1.3 If there is a match, replace S by S + C, and remove C. 576fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 577fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // First: Is a discontiguous contraction even possible? 578fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint16_t fcd16 = d->getFCD16(c); 579fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(fcd16 > 0xff); // The caller checked this already, as a shortcut. 580fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UChar32 nextCp = nextSkippedCodePoint(errorCode); 581fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(nextCp < 0) { 582fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // No further text. 583fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius backwardNumSkipped(1, errorCode); 584fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return ce32; 585fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 586fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ++lookAhead; 587fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint8_t prevCC = (uint8_t)fcd16; 588fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius fcd16 = d->getFCD16(nextCp); 589fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(fcd16 <= 0xff) { 590fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // The next code point after c is a starter (S2.1.1 "process each non-starter"). 591fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius backwardNumSkipped(2, errorCode); 592fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return ce32; 593fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 594fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 595fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // We have read and matched (lookAhead-2) code points, 596fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // read non-matching c and peeked ahead at nextCp. 597fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Return to the state before the mismatch and continue matching with nextCp. 598fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(skipped == NULL || skipped->isEmpty()) { 599fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(skipped == NULL) { 600fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius skipped = new SkippedState(); 601fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(skipped == NULL) { 602fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius errorCode = U_MEMORY_ALLOCATION_ERROR; 603fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return 0; 604fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 605fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 606fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius suffixes.reset(); 607fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(lookAhead > 2) { 608fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Replay the partial match so far. 609fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius backwardNumCodePoints(lookAhead, errorCode); 610fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius suffixes.firstForCodePoint(nextCodePoint(errorCode)); 611fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(int32_t i = 3; i < lookAhead; ++i) { 612fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius suffixes.nextForCodePoint(nextCodePoint(errorCode)); 613fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 614fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Skip c (which did not match) and nextCp (which we will try now). 615fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius forwardNumCodePoints(2, errorCode); 616fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 617fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius skipped->saveTrieState(suffixes); 618fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 619fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Reset to the trie state before the failed match of c. 620fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius skipped->resetToTrieState(suffixes); 621fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 622fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 623fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius skipped->setFirstSkipped(c); 624fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Number of code points read since the last match (at this point: c and nextCp). 625fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t sinceMatch = 2; 626fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius c = nextCp; 627fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(;;) { 628fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UStringTrieResult match; 629fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // "If C is not blocked from S, find if S + C has a match in the table." (S2.1.2) 630fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(prevCC < (fcd16 >> 8) && USTRINGTRIE_HAS_VALUE(match = suffixes.nextForCodePoint(c))) { 631fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // "If there is a match, replace S by S + C, and remove C." (S2.1.3) 632fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Keep prevCC unchanged. 633fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = (uint32_t)suffixes.getValue(); 634fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius sinceMatch = 0; 635fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius skipped->recordMatch(); 636fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(!USTRINGTRIE_HAS_NEXT(match)) { break; } 637fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius skipped->saveTrieState(suffixes); 638fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 639fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // No match for "S + C", skip C. 640fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius skipped->skip(c); 641fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius skipped->resetToTrieState(suffixes); 642fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius prevCC = (uint8_t)fcd16; 643fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 644fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if((c = nextSkippedCodePoint(errorCode)) < 0) { break; } 645fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ++sinceMatch; 646fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius fcd16 = d->getFCD16(c); 647fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(fcd16 <= 0xff) { 648fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // The next code point after c is a starter (S2.1.1 "process each non-starter"). 649fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 650fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 651fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 652fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius backwardNumSkipped(sinceMatch, errorCode); 653fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UBool isTopDiscontiguous = skipped->isEmpty(); 654fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius skipped->replaceMatch(); 655fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(isTopDiscontiguous && !skipped->isEmpty()) { 656fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // We did get a match after skipping one or more combining marks, 657fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // and we are not in a recursive discontiguous contraction. 658fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Append CEs from the contraction ce32 659fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // and then from the combining marks that we skipped before the match. 660fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius c = U_SENTINEL; 661fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(;;) { 662fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius appendCEsFromCE32(d, c, ce32, TRUE, errorCode); 663fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Fetch CE32s for skipped combining marks from the normal data, with fallback, 664fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // rather than from the CollationData where we found the contraction. 665fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(!skipped->hasNext()) { break; } 666fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius c = skipped->next(); 667fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = getDataCE32(c); 668fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ce32 == Collation::FALLBACK_CE32) { 669fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius d = data->base; 670fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = d->getCE32(c); 671fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 672fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius d = data; 673fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 674fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Note: A nested discontiguous-contraction match 675fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // replaces consumed combining marks with newly skipped ones 676fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // and resets the reading position to the beginning. 677fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 678fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius skipped->clear(); 679fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = Collation::NO_CE32; // Signal to the caller that the result is in the ceBuffer. 680fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 681fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return ce32; 682fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 683fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 684fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusvoid 685fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationIterator::appendNumericCEs(uint32_t ce32, UBool forward, UErrorCode &errorCode) { 686fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Collect digits. 687fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius CharString digits; 688fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(forward) { 689fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(;;) { 690fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius char digit = Collation::digitFromCE32(ce32); 691fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius digits.append(digit, errorCode); 692fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(numCpFwd == 0) { break; } 693fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UChar32 c = nextCodePoint(errorCode); 694fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(c < 0) { break; } 695fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = data->getCE32(c); 696fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ce32 == Collation::FALLBACK_CE32) { 697fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = data->base->getCE32(c); 698fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 699fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(!Collation::hasCE32Tag(ce32, Collation::DIGIT_TAG)) { 700fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius backwardNumCodePoints(1, errorCode); 701fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 702fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 703fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(numCpFwd > 0) { --numCpFwd; } 704fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 705fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 706fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(;;) { 707fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius char digit = Collation::digitFromCE32(ce32); 708fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius digits.append(digit, errorCode); 709fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UChar32 c = previousCodePoint(errorCode); 710fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(c < 0) { break; } 711fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = data->getCE32(c); 712fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ce32 == Collation::FALLBACK_CE32) { 713fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = data->base->getCE32(c); 714fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 715fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(!Collation::hasCE32Tag(ce32, Collation::DIGIT_TAG)) { 716fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius forwardNumCodePoints(1, errorCode); 717fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 718fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 719fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 720fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Reverse the digit string. 721fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius char *p = digits.data(); 722fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius char *q = p + digits.length() - 1; 723fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius while(p < q) { 724fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius char digit = *p; 725fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius *p++ = *q; 726fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius *q-- = digit; 727fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 728fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 729fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_FAILURE(errorCode)) { return; } 730fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t pos = 0; 731fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius do { 732fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Skip leading zeros. 733fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius while(pos < (digits.length() - 1) && digits[pos] == 0) { ++pos; } 734fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Write a sequence of CEs for at most 254 digits at a time. 735fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t segmentLength = digits.length() - pos; 736fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(segmentLength > 254) { segmentLength = 254; } 737fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius appendNumericSegmentCEs(digits.data() + pos, segmentLength, errorCode); 738fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius pos += segmentLength; 739fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } while(U_SUCCESS(errorCode) && pos < digits.length()); 740fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 741fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 742fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusvoid 743fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationIterator::appendNumericSegmentCEs(const char *digits, int32_t length, UErrorCode &errorCode) { 744fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(1 <= length && length <= 254); 745fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(length == 1 || digits[0] != 0); 746fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t numericPrimary = data->numericPrimary; 747fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Note: We use primary byte values 2..255: digits are not compressible. 748fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(length <= 7) { 749fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Very dense encoding for small numbers. 750fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t value = digits[0]; 751fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius for(int32_t i = 1; i < length; ++i) { 752fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius value = value * 10 + digits[i]; 753fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 754fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Primary weight second byte values: 755fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // 74 byte values 2.. 75 for small numbers in two-byte primary weights. 756fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // 40 byte values 76..115 for medium numbers in three-byte primary weights. 757fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // 16 byte values 116..131 for large numbers in four-byte primary weights. 758fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // 124 byte values 132..255 for very large numbers with 4..127 digit pairs. 759fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t firstByte = 2; 760fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t numBytes = 74; 761fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(value < numBytes) { 762fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Two-byte primary for 0..73, good for day & month numbers etc. 763fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t primary = numericPrimary | ((firstByte + value) << 16); 764fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ceBuffer.append(Collation::makeCE(primary), errorCode); 765fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return; 766fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 767fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius value -= numBytes; 768fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius firstByte += numBytes; 769fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius numBytes = 40; 770fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(value < numBytes * 254) { 771fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Three-byte primary for 74..10233=74+40*254-1, good for year numbers and more. 772fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t primary = numericPrimary | 773fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ((firstByte + value / 254) << 16) | ((2 + value % 254) << 8); 774fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ceBuffer.append(Collation::makeCE(primary), errorCode); 775fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return; 776fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 777fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius value -= numBytes * 254; 778fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius firstByte += numBytes; 779fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius numBytes = 16; 780fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(value < numBytes * 254 * 254) { 781fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Four-byte primary for 10234..1042489=10234+16*254*254-1. 782fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t primary = numericPrimary | (2 + value % 254); 783fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius value /= 254; 784fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius primary |= (2 + value % 254) << 8; 785fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius value /= 254; 786fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius primary |= (firstByte + value % 254) << 16; 787fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ceBuffer.append(Collation::makeCE(primary), errorCode); 788fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return; 789fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 790fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // original value > 1042489 791fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 792fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(length >= 7); 793fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 794fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // The second primary byte value 132..255 indicates the number of digit pairs (4..127), 795fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // then we generate primary bytes with those pairs. 796fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Omit trailing 00 pairs. 797fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Decrement the value for the last pair. 798fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 799fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Set the exponent. 4 pairs->132, 5 pairs->133, ..., 127 pairs->255. 800fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t numPairs = (length + 1) / 2; 801fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t primary = numericPrimary | ((132 - 4 + numPairs) << 16); 802fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Find the length without trailing 00 pairs. 803fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius while(digits[length - 1] == 0 && digits[length - 2] == 0) { 804fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius length -= 2; 805fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 806fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Read the first pair. 807fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t pair; 808fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t pos; 809fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(length & 1) { 810fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Only "half a pair" if we have an odd number of digits. 811fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius pair = digits[0]; 812fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius pos = 1; 813fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 814fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius pair = digits[0] * 10 + digits[1]; 815fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius pos = 2; 816fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 817fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius pair = 11 + 2 * pair; 818fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Add the pairs of digits between pos and length. 819fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t shift = 8; 820fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius while(pos < length) { 821fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(shift == 0) { 822fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Every three pairs/bytes we need to store a 4-byte-primary CE 823fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // and start with a new CE with the '0' primary lead byte. 824fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius primary |= pair; 825fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ceBuffer.append(Collation::makeCE(primary), errorCode); 826fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius primary = numericPrimary; 827fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius shift = 16; 828fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 829fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius primary |= pair << shift; 830fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius shift -= 8; 831fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 832fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius pair = 11 + 2 * (digits[pos] * 10 + digits[pos + 1]); 833fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius pos += 2; 834fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 835fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius primary |= (pair - 1) << shift; 836fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ceBuffer.append(Collation::makeCE(primary), errorCode); 837fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 838fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 839fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusint64_t 840fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationIterator::previousCE(UVector32 &offsets, UErrorCode &errorCode) { 841fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ceBuffer.length > 0) { 842fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Return the previous buffered CE. 843fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return ceBuffer.get(--ceBuffer.length); 844fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 845fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius offsets.removeAllElements(); 846fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t limitOffset = getOffset(); 847fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius UChar32 c = previousCodePoint(errorCode); 848fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(c < 0) { return Collation::NO_CE; } 849fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(data->isUnsafeBackward(c, isNumeric)) { 850fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return previousCEUnsafe(c, offsets, errorCode); 851fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 852fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Simple, safe-backwards iteration: 853fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Get a CE going backwards, handle prefixes but no contractions. 854fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius uint32_t ce32 = data->getCE32(c); 855fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius const CollationData *d; 856fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ce32 == Collation::FALLBACK_CE32) { 857fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius d = data->base; 858fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ce32 = d->getCE32(c); 859fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 860fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius d = data; 861fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 862fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(Collation::isSimpleOrLongCE32(ce32)) { 863fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return Collation::ceFromCE32(ce32); 864fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 865fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius appendCEsFromCE32(d, c, ce32, FALSE, errorCode); 866fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_SUCCESS(errorCode)) { 867fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(ceBuffer.length > 1) { 868fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius offsets.addElement(getOffset(), errorCode); 869fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // For an expansion, the offset of each non-initial CE is the limit offset, 870fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // consistent with forward iteration. 871fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius while(offsets.size() <= ceBuffer.length) { 872fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius offsets.addElement(limitOffset, errorCode); 873fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius }; 874fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 875fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return ceBuffer.get(--ceBuffer.length); 876fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 877fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return Collation::NO_CE_PRIMARY; 878fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 879fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 880fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 881fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusint64_t 882fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusCollationIterator::previousCEUnsafe(UChar32 c, UVector32 &offsets, UErrorCode &errorCode) { 883fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // We just move through the input counting safe and unsafe code points 884fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // without collecting the unsafe-backward substring into a buffer and 885fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // switching to it. 886fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // This is to keep the logic simple. Otherwise we would have to handle 887fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // prefix matching going before the backward buffer, switching 888fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // to iteration and back, etc. 889fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // In the most important case of iterating over a normal string, 890fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // reading from the string itself is already maximally fast. 891fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // The only drawback there is that after getting the CEs we always 892fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // skip backward to the safe character rather than switching out 893fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // of a backwardBuffer. 894fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // But this should not be the common case for previousCE(), 895fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // and correctness and maintainability are more important than 896fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // complex optimizations. 897fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Find the first safe character before c. 898fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t numBackward = 1; 899fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius while((c = previousCodePoint(errorCode)) >= 0) { 900fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius ++numBackward; 901fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(!data->isUnsafeBackward(c, isNumeric)) { 902fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius break; 903fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 904fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 905fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Set the forward iteration limit. 906fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Note: This counts code points. 907fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // We cannot enforce a limit in the middle of a surrogate pair or similar. 908fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius numCpFwd = numBackward; 909fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Reset the forward iterator. 910fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius cesIndex = 0; 911fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(ceBuffer.length == 0); 912fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Go forward and collect the CEs. 913fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius int32_t offset = getOffset(); 914fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius while(numCpFwd > 0) { 915fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // nextCE() normally reads one code point. 916fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Contraction matching and digit specials read more and check numCpFwd. 917fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius --numCpFwd; 918fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Append one or more CEs to the ceBuffer. 919fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius (void)nextCE(errorCode); 920fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(U_FAILURE(errorCode) || ceBuffer.get(ceBuffer.length - 1) != Collation::NO_CE); 921fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // No need to loop for getting each expansion CE from nextCE(). 922fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius cesIndex = ceBuffer.length; 923fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // However, we need to write an offset for each CE. 924fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // This is for CollationElementIterator::getOffset() to return 925fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // intermediate offsets from the unsafe-backwards segment. 926fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(offsets.size() < ceBuffer.length); 927fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius offsets.addElement(offset, errorCode); 928fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // For an expansion, the offset of each non-initial CE is the limit offset, 929fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // consistent with forward iteration. 930fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius offset = getOffset(); 931fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius while(offsets.size() < ceBuffer.length) { 932fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius offsets.addElement(offset, errorCode); 933fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius }; 934fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 935fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius U_ASSERT(offsets.size() == ceBuffer.length); 936fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // End offset corresponding to just after the unsafe-backwards segment. 937fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius offsets.addElement(offset, errorCode); 938fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Reset the forward iteration limit 939fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // and move backward to before the segment for which we fetched CEs. 940fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius numCpFwd = -1; 941fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius backwardNumCodePoints(numBackward, errorCode); 942fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius // Use the collected CEs and return the last one. 943fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius cesIndex = 0; // Avoid cesIndex > ceBuffer.length when that gets decremented. 944fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius if(U_SUCCESS(errorCode)) { 945fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return ceBuffer.get(--ceBuffer.length); 946fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } else { 947fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius return Collation::NO_CE_PRIMARY; 948fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius } 949fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius} 950fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 951fceb39872958b9fa2505e63f8b8699a9e0f882f4ccorneliusU_NAMESPACE_END 952fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius 953fceb39872958b9fa2505e63f8b8699a9e0f882f4ccornelius#endif // !UCONFIG_NO_COLLATION 954