Lines Matching defs:tokens
125 * Convert the input to a array of tokens. The sequence of ASCII or Unknown characters without
130 ArrayList<Token> tokens = new ArrayList<Token>();
132 // return empty tokens.
133 return tokens;
149 addToken(sb, tokens, tokenType);
155 addToken(sb, tokens, tokenType);
157 tokens.add(token);
161 addToken(sb, tokens, tokenType);
169 addToken(sb, tokens, tokenType);
171 return tokens;
175 final StringBuilder sb, final ArrayList<Token> tokens, final int tokenType) {
177 tokens.add(new Token(tokenType, str, str));