1// Copyright 2013 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include "chrome/browser/ui/app_list/search/tokenized_string.h" 6 7#include "base/i18n/break_iterator.h" 8#include "base/i18n/case_conversion.h" 9#include "base/logging.h" 10#include "chrome/browser/ui/app_list/search/term_break_iterator.h" 11 12using base::i18n::BreakIterator; 13 14namespace app_list { 15 16TokenizedString::TokenizedString(const string16& text) 17 : text_(text) { 18 Tokenize(); 19} 20 21TokenizedString::~TokenizedString() {} 22 23void TokenizedString::Tokenize() { 24 BreakIterator break_iter(text_, BreakIterator::BREAK_WORD); 25 CHECK(break_iter.Init()); 26 27 while (break_iter.Advance()) { 28 if (!break_iter.IsWord()) 29 continue; 30 31 const string16 word(break_iter.GetString()); 32 const size_t word_start = break_iter.prev(); 33 TermBreakIterator term_iter(word); 34 while (term_iter.Advance()) { 35 tokens_.push_back(base::i18n::ToLower(term_iter.GetCurrentTerm())); 36 mappings_.push_back(ui::Range(word_start + term_iter.prev(), 37 word_start + term_iter.pos())); 38 } 39 } 40} 41 42} // namespace app_list 43