Lines Matching defs:words
19 // ngram_size: num of words for each output item.
20 // max_skip_size: max num of words to skip.
29 // A list of strings, each of which contains ngram_size words.
91 // Split sentence to words.
92 std::vector<StringRef> words;
98 words.push_back({strref.str + prev_idx, i - prev_idx});
104 words.push_back({strref.str + prev_idx, strref.len - prev_idx});
109 if (words.size() < params->ngram_size) {
119 int num_words = words.size();
134 // words to generate the ngram.
137 gram[i] = words[stack[i]];