1/* 2 ************************************************************************* 3 * COPYRIGHT: 4 * Copyright (c) 1996-2012, International Business Machines Corporation and 5 * others. All Rights Reserved. 6 ************************************************************************* 7 */ 8 9#include "unicode/utypes.h" 10 11#if !UCONFIG_NO_NORMALIZATION 12 13#include "unicode/uniset.h" 14#include "unicode/unistr.h" 15#include "unicode/chariter.h" 16#include "unicode/schriter.h" 17#include "unicode/uchriter.h" 18#include "unicode/normlzr.h" 19#include "unicode/utf16.h" 20#include "cmemory.h" 21#include "normalizer2impl.h" 22#include "uprops.h" // for uniset_getUnicode32Instance() 23 24U_NAMESPACE_BEGIN 25 26UOBJECT_DEFINE_RTTI_IMPLEMENTATION(Normalizer) 27 28//------------------------------------------------------------------------- 29// Constructors and other boilerplate 30//------------------------------------------------------------------------- 31 32Normalizer::Normalizer(const UnicodeString& str, UNormalizationMode mode) : 33 UObject(), fFilteredNorm2(NULL), fNorm2(NULL), fUMode(mode), fOptions(0), 34 text(new StringCharacterIterator(str)), 35 currentIndex(0), nextIndex(0), 36 buffer(), bufferPos(0) 37{ 38 init(); 39} 40 41Normalizer::Normalizer(const UChar *str, int32_t length, UNormalizationMode mode) : 42 UObject(), fFilteredNorm2(NULL), fNorm2(NULL), fUMode(mode), fOptions(0), 43 text(new UCharCharacterIterator(str, length)), 44 currentIndex(0), nextIndex(0), 45 buffer(), bufferPos(0) 46{ 47 init(); 48} 49 50Normalizer::Normalizer(const CharacterIterator& iter, UNormalizationMode mode) : 51 UObject(), fFilteredNorm2(NULL), fNorm2(NULL), fUMode(mode), fOptions(0), 52 text(iter.clone()), 53 currentIndex(0), nextIndex(0), 54 buffer(), bufferPos(0) 55{ 56 init(); 57} 58 59Normalizer::Normalizer(const Normalizer ©) : 60 UObject(copy), fFilteredNorm2(NULL), fNorm2(NULL), fUMode(copy.fUMode), fOptions(copy.fOptions), 61 text(copy.text->clone()), 62 currentIndex(copy.currentIndex), nextIndex(copy.nextIndex), 63 buffer(copy.buffer), bufferPos(copy.bufferPos) 64{ 65 init(); 66} 67 68void 69Normalizer::init() { 70 UErrorCode errorCode=U_ZERO_ERROR; 71 fNorm2=Normalizer2Factory::getInstance(fUMode, errorCode); 72 if(fOptions&UNORM_UNICODE_3_2) { 73 delete fFilteredNorm2; 74 fNorm2=fFilteredNorm2= 75 new FilteredNormalizer2(*fNorm2, *uniset_getUnicode32Instance(errorCode)); 76 } 77 if(U_FAILURE(errorCode)) { 78 errorCode=U_ZERO_ERROR; 79 fNorm2=Normalizer2Factory::getNoopInstance(errorCode); 80 } 81} 82 83Normalizer::~Normalizer() 84{ 85 delete fFilteredNorm2; 86 delete text; 87} 88 89Normalizer* 90Normalizer::clone() const 91{ 92 return new Normalizer(*this); 93} 94 95/** 96 * Generates a hash code for this iterator. 97 */ 98int32_t Normalizer::hashCode() const 99{ 100 return text->hashCode() + fUMode + fOptions + buffer.hashCode() + bufferPos + currentIndex + nextIndex; 101} 102 103UBool Normalizer::operator==(const Normalizer& that) const 104{ 105 return 106 this==&that || 107 (fUMode==that.fUMode && 108 fOptions==that.fOptions && 109 *text==*that.text && 110 buffer==that.buffer && 111 bufferPos==that.bufferPos && 112 nextIndex==that.nextIndex); 113} 114 115//------------------------------------------------------------------------- 116// Static utility methods 117//------------------------------------------------------------------------- 118 119void U_EXPORT2 120Normalizer::normalize(const UnicodeString& source, 121 UNormalizationMode mode, int32_t options, 122 UnicodeString& result, 123 UErrorCode &status) { 124 if(source.isBogus() || U_FAILURE(status)) { 125 result.setToBogus(); 126 if(U_SUCCESS(status)) { 127 status=U_ILLEGAL_ARGUMENT_ERROR; 128 } 129 } else { 130 UnicodeString localDest; 131 UnicodeString *dest; 132 133 if(&source!=&result) { 134 dest=&result; 135 } else { 136 // the source and result strings are the same object, use a temporary one 137 dest=&localDest; 138 } 139 const Normalizer2 *n2=Normalizer2Factory::getInstance(mode, status); 140 if(U_SUCCESS(status)) { 141 if(options&UNORM_UNICODE_3_2) { 142 FilteredNormalizer2(*n2, *uniset_getUnicode32Instance(status)). 143 normalize(source, *dest, status); 144 } else { 145 n2->normalize(source, *dest, status); 146 } 147 } 148 if(dest==&localDest && U_SUCCESS(status)) { 149 result=*dest; 150 } 151 } 152} 153 154void U_EXPORT2 155Normalizer::compose(const UnicodeString& source, 156 UBool compat, int32_t options, 157 UnicodeString& result, 158 UErrorCode &status) { 159 normalize(source, compat ? UNORM_NFKC : UNORM_NFC, options, result, status); 160} 161 162void U_EXPORT2 163Normalizer::decompose(const UnicodeString& source, 164 UBool compat, int32_t options, 165 UnicodeString& result, 166 UErrorCode &status) { 167 normalize(source, compat ? UNORM_NFKD : UNORM_NFD, options, result, status); 168} 169 170UNormalizationCheckResult 171Normalizer::quickCheck(const UnicodeString& source, 172 UNormalizationMode mode, int32_t options, 173 UErrorCode &status) { 174 const Normalizer2 *n2=Normalizer2Factory::getInstance(mode, status); 175 if(U_SUCCESS(status)) { 176 if(options&UNORM_UNICODE_3_2) { 177 return FilteredNormalizer2(*n2, *uniset_getUnicode32Instance(status)). 178 quickCheck(source, status); 179 } else { 180 return n2->quickCheck(source, status); 181 } 182 } else { 183 return UNORM_MAYBE; 184 } 185} 186 187UBool 188Normalizer::isNormalized(const UnicodeString& source, 189 UNormalizationMode mode, int32_t options, 190 UErrorCode &status) { 191 const Normalizer2 *n2=Normalizer2Factory::getInstance(mode, status); 192 if(U_SUCCESS(status)) { 193 if(options&UNORM_UNICODE_3_2) { 194 return FilteredNormalizer2(*n2, *uniset_getUnicode32Instance(status)). 195 isNormalized(source, status); 196 } else { 197 return n2->isNormalized(source, status); 198 } 199 } else { 200 return FALSE; 201 } 202} 203 204UnicodeString & U_EXPORT2 205Normalizer::concatenate(const UnicodeString &left, const UnicodeString &right, 206 UnicodeString &result, 207 UNormalizationMode mode, int32_t options, 208 UErrorCode &errorCode) { 209 if(left.isBogus() || right.isBogus() || U_FAILURE(errorCode)) { 210 result.setToBogus(); 211 if(U_SUCCESS(errorCode)) { 212 errorCode=U_ILLEGAL_ARGUMENT_ERROR; 213 } 214 } else { 215 UnicodeString localDest; 216 UnicodeString *dest; 217 218 if(&right!=&result) { 219 dest=&result; 220 } else { 221 // the right and result strings are the same object, use a temporary one 222 dest=&localDest; 223 } 224 *dest=left; 225 const Normalizer2 *n2=Normalizer2Factory::getInstance(mode, errorCode); 226 if(U_SUCCESS(errorCode)) { 227 if(options&UNORM_UNICODE_3_2) { 228 FilteredNormalizer2(*n2, *uniset_getUnicode32Instance(errorCode)). 229 append(*dest, right, errorCode); 230 } else { 231 n2->append(*dest, right, errorCode); 232 } 233 } 234 if(dest==&localDest && U_SUCCESS(errorCode)) { 235 result=*dest; 236 } 237 } 238 return result; 239} 240 241//------------------------------------------------------------------------- 242// Iteration API 243//------------------------------------------------------------------------- 244 245/** 246 * Return the current character in the normalized text. 247 */ 248UChar32 Normalizer::current() { 249 if(bufferPos<buffer.length() || nextNormalize()) { 250 return buffer.char32At(bufferPos); 251 } else { 252 return DONE; 253 } 254} 255 256/** 257 * Return the next character in the normalized text and advance 258 * the iteration position by one. If the end 259 * of the text has already been reached, {@link #DONE} is returned. 260 */ 261UChar32 Normalizer::next() { 262 if(bufferPos<buffer.length() || nextNormalize()) { 263 UChar32 c=buffer.char32At(bufferPos); 264 bufferPos+=U16_LENGTH(c); 265 return c; 266 } else { 267 return DONE; 268 } 269} 270 271/** 272 * Return the previous character in the normalized text and decrement 273 * the iteration position by one. If the beginning 274 * of the text has already been reached, {@link #DONE} is returned. 275 */ 276UChar32 Normalizer::previous() { 277 if(bufferPos>0 || previousNormalize()) { 278 UChar32 c=buffer.char32At(bufferPos-1); 279 bufferPos-=U16_LENGTH(c); 280 return c; 281 } else { 282 return DONE; 283 } 284} 285 286void Normalizer::reset() { 287 currentIndex=nextIndex=text->setToStart(); 288 clearBuffer(); 289} 290 291void 292Normalizer::setIndexOnly(int32_t index) { 293 text->setIndex(index); // pins index 294 currentIndex=nextIndex=text->getIndex(); 295 clearBuffer(); 296} 297 298/** 299 * Return the first character in the normalized text. This resets 300 * the <tt>Normalizer's</tt> position to the beginning of the text. 301 */ 302UChar32 Normalizer::first() { 303 reset(); 304 return next(); 305} 306 307/** 308 * Return the last character in the normalized text. This resets 309 * the <tt>Normalizer's</tt> position to be just before the 310 * the input text corresponding to that normalized character. 311 */ 312UChar32 Normalizer::last() { 313 currentIndex=nextIndex=text->setToEnd(); 314 clearBuffer(); 315 return previous(); 316} 317 318/** 319 * Retrieve the current iteration position in the input text that is 320 * being normalized. This method is useful in applications such as 321 * searching, where you need to be able to determine the position in 322 * the input text that corresponds to a given normalized output character. 323 * <p> 324 * <b>Note:</b> This method sets the position in the <em>input</em>, while 325 * {@link #next} and {@link #previous} iterate through characters in the 326 * <em>output</em>. This means that there is not necessarily a one-to-one 327 * correspondence between characters returned by <tt>next</tt> and 328 * <tt>previous</tt> and the indices passed to and returned from 329 * <tt>setIndex</tt> and {@link #getIndex}. 330 * 331 */ 332int32_t Normalizer::getIndex() const { 333 if(bufferPos<buffer.length()) { 334 return currentIndex; 335 } else { 336 return nextIndex; 337 } 338} 339 340/** 341 * Retrieve the index of the start of the input text. This is the begin index 342 * of the <tt>CharacterIterator</tt> or the start (i.e. 0) of the <tt>String</tt> 343 * over which this <tt>Normalizer</tt> is iterating 344 */ 345int32_t Normalizer::startIndex() const { 346 return text->startIndex(); 347} 348 349/** 350 * Retrieve the index of the end of the input text. This is the end index 351 * of the <tt>CharacterIterator</tt> or the length of the <tt>String</tt> 352 * over which this <tt>Normalizer</tt> is iterating 353 */ 354int32_t Normalizer::endIndex() const { 355 return text->endIndex(); 356} 357 358//------------------------------------------------------------------------- 359// Property access methods 360//------------------------------------------------------------------------- 361 362void 363Normalizer::setMode(UNormalizationMode newMode) 364{ 365 fUMode = newMode; 366 init(); 367} 368 369UNormalizationMode 370Normalizer::getUMode() const 371{ 372 return fUMode; 373} 374 375void 376Normalizer::setOption(int32_t option, 377 UBool value) 378{ 379 if (value) { 380 fOptions |= option; 381 } else { 382 fOptions &= (~option); 383 } 384 init(); 385} 386 387UBool 388Normalizer::getOption(int32_t option) const 389{ 390 return (fOptions & option) != 0; 391} 392 393/** 394 * Set the input text over which this <tt>Normalizer</tt> will iterate. 395 * The iteration position is set to the beginning of the input text. 396 */ 397void 398Normalizer::setText(const UnicodeString& newText, 399 UErrorCode &status) 400{ 401 if (U_FAILURE(status)) { 402 return; 403 } 404 CharacterIterator *newIter = new StringCharacterIterator(newText); 405 if (newIter == NULL) { 406 status = U_MEMORY_ALLOCATION_ERROR; 407 return; 408 } 409 delete text; 410 text = newIter; 411 reset(); 412} 413 414/** 415 * Set the input text over which this <tt>Normalizer</tt> will iterate. 416 * The iteration position is set to the beginning of the string. 417 */ 418void 419Normalizer::setText(const CharacterIterator& newText, 420 UErrorCode &status) 421{ 422 if (U_FAILURE(status)) { 423 return; 424 } 425 CharacterIterator *newIter = newText.clone(); 426 if (newIter == NULL) { 427 status = U_MEMORY_ALLOCATION_ERROR; 428 return; 429 } 430 delete text; 431 text = newIter; 432 reset(); 433} 434 435void 436Normalizer::setText(const UChar* newText, 437 int32_t length, 438 UErrorCode &status) 439{ 440 if (U_FAILURE(status)) { 441 return; 442 } 443 CharacterIterator *newIter = new UCharCharacterIterator(newText, length); 444 if (newIter == NULL) { 445 status = U_MEMORY_ALLOCATION_ERROR; 446 return; 447 } 448 delete text; 449 text = newIter; 450 reset(); 451} 452 453/** 454 * Copies the text under iteration into the UnicodeString referred to by "result". 455 * @param result Receives a copy of the text under iteration. 456 */ 457void 458Normalizer::getText(UnicodeString& result) 459{ 460 text->getText(result); 461} 462 463//------------------------------------------------------------------------- 464// Private utility methods 465//------------------------------------------------------------------------- 466 467void Normalizer::clearBuffer() { 468 buffer.remove(); 469 bufferPos=0; 470} 471 472UBool 473Normalizer::nextNormalize() { 474 clearBuffer(); 475 currentIndex=nextIndex; 476 text->setIndex(nextIndex); 477 if(!text->hasNext()) { 478 return FALSE; 479 } 480 // Skip at least one character so we make progress. 481 UnicodeString segment(text->next32PostInc()); 482 while(text->hasNext()) { 483 UChar32 c; 484 if(fNorm2->hasBoundaryBefore(c=text->next32PostInc())) { 485 text->move32(-1, CharacterIterator::kCurrent); 486 break; 487 } 488 segment.append(c); 489 } 490 nextIndex=text->getIndex(); 491 UErrorCode errorCode=U_ZERO_ERROR; 492 fNorm2->normalize(segment, buffer, errorCode); 493 return U_SUCCESS(errorCode) && !buffer.isEmpty(); 494} 495 496UBool 497Normalizer::previousNormalize() { 498 clearBuffer(); 499 nextIndex=currentIndex; 500 text->setIndex(currentIndex); 501 if(!text->hasPrevious()) { 502 return FALSE; 503 } 504 UnicodeString segment; 505 while(text->hasPrevious()) { 506 UChar32 c=text->previous32(); 507 segment.insert(0, c); 508 if(fNorm2->hasBoundaryBefore(c)) { 509 break; 510 } 511 } 512 currentIndex=text->getIndex(); 513 UErrorCode errorCode=U_ZERO_ERROR; 514 fNorm2->normalize(segment, buffer, errorCode); 515 bufferPos=buffer.length(); 516 return U_SUCCESS(errorCode) && !buffer.isEmpty(); 517} 518 519U_NAMESPACE_END 520 521#endif /* #if !UCONFIG_NO_NORMALIZATION */ 522