1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "chrome/browser/ui/app_list/search/tokenized_string.h"
6
7#include "base/i18n/break_iterator.h"
8#include "base/i18n/case_conversion.h"
9#include "base/logging.h"
10#include "chrome/browser/ui/app_list/search/term_break_iterator.h"
11
12using base::i18n::BreakIterator;
13
14namespace app_list {
15
16TokenizedString::TokenizedString(const string16& text)
17    : text_(text) {
18  Tokenize();
19}
20
21TokenizedString::~TokenizedString() {}
22
23void TokenizedString::Tokenize() {
24  BreakIterator break_iter(text_,  BreakIterator::BREAK_WORD);
25  CHECK(break_iter.Init());
26
27  while (break_iter.Advance()) {
28    if (!break_iter.IsWord())
29      continue;
30
31    const string16 word(break_iter.GetString());
32    const size_t word_start = break_iter.prev();
33    TermBreakIterator term_iter(word);
34    while (term_iter.Advance()) {
35      tokens_.push_back(base::i18n::ToLower(term_iter.GetCurrentTerm()));
36      mappings_.push_back(ui::Range(word_start + term_iter.prev(),
37                                    word_start + term_iter.pos()));
38    }
39  }
40}
41
42}  // namespace app_list
43