break_iterator.h revision f8ee788a64d60abd8f2d742a5fdedde054ecd910
15c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Copyright (c) 2011 The Chromium Authors. All rights reserved. 25c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Use of this source code is governed by a BSD-style license that can be 35c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// found in the LICENSE file. 45c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 55c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#ifndef BASE_I18N_BREAK_ITERATOR_H_ 65c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#define BASE_I18N_BREAK_ITERATOR_H_ 75c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 85c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include "base/basictypes.h" 95c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include "base/i18n/base_i18n_export.h" 105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include "base/strings/string16.h" 115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// The BreakIterator class iterates through the words, word breaks, and 135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// line breaks in a UTF-16 string. 145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// It provides several modes, BREAK_WORD, BREAK_LINE, and BREAK_NEWLINE, 165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// which modify how characters are aggregated into the returned string. 175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Under BREAK_WORD mode, once a word is encountered any non-word 195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// characters are not included in the returned string (e.g. in the 205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// UTF-16 equivalent of the string " foo bar! ", the word breaks are at 215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// the periods in ". .foo. .bar.!. ."). 225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Note that Chinese/Japanese/Thai do not use spaces between words so that 235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// boundaries can fall in the middle of a continuous run of non-space / 245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// non-punctuation characters. 255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Under BREAK_LINE mode, once a line breaking opportunity is encountered, 275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// any non-word characters are included in the returned string, breaking 285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// only when a space-equivalent character or a line breaking opportunity 295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// is encountered (e.g. in the UTF16-equivalent of the string " foo bar! ", 305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// the breaks are at the periods in ". .foo .bar! ."). 315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// 325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// Note that lines can be broken at any character/syllable/grapheme cluster 335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// boundary in Chinese/Japanese/Korean and at word boundaries in Thai 345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// (Thai does not use spaces between words). Therefore, this is NOT the same 3551b2906e11752df6c18351cf520e30522d3b53a1Torne (Richard Coles)// as breaking only at space-equivalent characters where its former 361e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles)// name (BREAK_SPACE) implied. 3709380295ba73501a205346becac22c6978e4671dTorne (Richard Coles)// 38521d96ec04ace82590870fb04353ec4f82bb150fTorne (Richard Coles)// Under BREAK_NEWLINE mode, all characters are included in the returned 39521d96ec04ace82590870fb04353ec4f82bb150fTorne (Richard Coles)// string, breaking only when a newline-equivalent character is encountered 40521d96ec04ace82590870fb04353ec4f82bb150fTorne (Richard Coles)// (eg. in the UTF-16 equivalent of the string "foo\nbar!\n\n", the line 41c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles)// breaks are at the periods in ".foo\n.bar\n.\n."). 42521d96ec04ace82590870fb04353ec4f82bb150fTorne (Richard Coles)// 43521d96ec04ace82590870fb04353ec4f82bb150fTorne (Richard Coles)// To extract the words from a string, move a BREAK_WORD BreakIterator 445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// through the string and test whether IsWord() is true. E.g., 45c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles)// BreakIterator iter(str, BreakIterator::BREAK_WORD); 465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// if (!iter.Init()) 4709380295ba73501a205346becac22c6978e4671dTorne (Richard Coles)// return false; 485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// while (iter.Advance()) { 495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// if (iter.IsWord()) { 505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// // Region [iter.prev(), iter.pos()) contains a word. 515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// VLOG(1) << "word: " << iter.GetString(); 525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// } 535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)// } 545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)namespace base { 565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)namespace i18n { 575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class BASE_I18N_EXPORT BreakIterator { 595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) public: 605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) enum BreakType { 615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) BREAK_WORD, 625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) BREAK_LINE, 635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // TODO(jshin): Remove this after reviewing call sites. 645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // If call sites really need break only on space-like characters 655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // implement it separately. 665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) BREAK_SPACE = BREAK_LINE, 675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) BREAK_NEWLINE, 685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) BREAK_CHARACTER, 695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // But don't remove this one! 705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) RULE_BASED, 71926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) }; 725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Requires |str| to live as long as the BreakIterator does. 745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) BreakIterator(const string16& str, BreakType break_type); 755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Make a rule-based iterator. BreakType == RULE_BASED is implied. 765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // TODO(andrewhayden): This signature could easily be misinterpreted as 775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // "(const string16& str, const string16& locale)". We should do something 785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // better. 795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) BreakIterator(const string16& str, const string16& rules); 805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ~BreakIterator(); 815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Init() must be called before any of the iterators are valid. 8353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) // Returns false if ICU failed to initialize. 8409380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) bool Init(); 855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 8609380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) // Advance to the next break. Returns false if we've run past the end of 8709380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) // the string. (Note that the very last "break" is after the final 88521d96ec04ace82590870fb04353ec4f82bb150fTorne (Richard Coles) // character in the string, and when we advance to that position it's the 89926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // last time Advance() returns true.) 90926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) bool Advance(); 91926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 92926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // Updates the text used by the iterator, resetting the iterator as if 937242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci // if Init() had been called again. Any old state is lost. Returns true 94926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // unless there is an error setting the text. 957242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci bool SetText(const base::char16* text, const size_t length); 96926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Under BREAK_WORD mode, returns true if the break we just hit is the 98926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // end of a word. (Otherwise, the break iterator just skipped over e.g. 995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // whitespace or punctuation.) Under BREAK_LINE and BREAK_NEWLINE modes, 1005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // this distinction doesn't apply and it always returns false. 1015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) bool IsWord() const; 1025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Under BREAK_WORD mode, returns true if |position| is at the end of word or 104926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) // at the start of word. It always returns false under BREAK_LINE and 1055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // BREAK_NEWLINE modes. 106926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) bool IsEndOfWord(size_t position) const; 1077242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci bool IsStartOfWord(size_t position) const; 10853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 10909380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) // Returns the string between prev() and pos(). 1105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Advance() must have been called successfully at least once for pos() to 1115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // have advanced to somewhere useful. 1125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) string16 GetString() const; 1135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 114c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) // Returns the value of pos() returned before Advance() was last called. 1155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t prev() const { return prev_; } 1165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 117 // Returns the current break position within the string, 118 // or BreakIterator::npos when done. 119 size_t pos() const { return pos_; } 120 121 private: 122 // ICU iterator, avoiding ICU ubrk.h dependence. 123 // This is actually an ICU UBreakiterator* type, which turns out to be 124 // a typedef for a void* in the ICU headers. Using void* directly prevents 125 // callers from needing access to the ICU public headers directory. 126 void* iter_; 127 128 // The string we're iterating over. Can be changed with SetText(...) 129 const string16& string_; 130 131 // Rules for our iterator. Mutually exclusive with break_type_. 132 const string16 rules_; 133 134 // The breaking style (word/space/newline). Mutually exclusive with rules_ 135 BreakType break_type_; 136 137 // Previous and current iterator positions. 138 size_t prev_, pos_; 139 140 DISALLOW_COPY_AND_ASSIGN(BreakIterator); 141}; 142 143} // namespace i18n 144} // namespace base 145 146#endif // BASE_I18N_BREAK_ITERATOR_H_ 147