Lines Matching refs:Tokenizer
51 class Tokenizer;
81 class LIBPROTOBUF_EXPORT Tokenizer {
83 // Construct a Tokenizer that reads and tokenizes text from the given
86 Tokenizer(ZeroCopyInputStream* input, ErrorCollector* error_collector);
87 ~Tokenizer();
138 // comes from a TYPE_FLOAT token parsed by Tokenizer. If it doesn't, the
143 // comes from a TYPE_STRING token parsed by Tokenizer. If it doesn't, the
153 // parsed by a Tokenizer, the result is undefined (possibly an assert
180 GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Tokenizer);
290 inline const Tokenizer::Token& Tokenizer::current() {
294 inline void Tokenizer::ParseString(const string& text, string* output) {