Lines Matching refs:token
60 * Type of this token, ASCII, PINYIN or UNKNOWN.
98 private void tokenize(char character, Token token) {
99 token.source = Character.toString(character);
103 token.type = Token.LATIN;
104 token.target = token.source;
110 token.type = Token.LATIN;
111 token.target = mAsciiTransliterator == null ? token.source :
112 mAsciiTransliterator.transliterate(token.source);
116 token.type = Token.PINYIN;
117 token.target = mPinyinTransliterator.transliterate(token.source);
118 if (TextUtils.isEmpty(token.target) ||
119 TextUtils.equals(token.source, token.target)) {
120 token.type = Token.UNKNOWN;
121 token.target = token.source;
135 * Token. If there is no Chinese transliterator, the empty token array is returned.
147 Token token = new Token();
149 // Go through the input, create a new token when
160 tokenize(character, token);
161 if (token.type == Token.PINYIN) {
165 tokens.add(token);
166 token = new Token();
168 if (tokenType != token.type && sb.length() > 0) {
171 sb.append(token.target);
173 tokenType = token.type;