Searched refs:Tokenizer (Results 1 - 25 of 86) sorted by relevance

1234

/external/protobuf/src/google/protobuf/io/
H A Dtokenizer_unittest.cc183 EXPECT_TRUE(Tokenizer::ParseInteger(text, kuint64max, &result));
198 Tokenizer::TokenType type;
208 { "hello", Tokenizer::TYPE_IDENTIFIER },
211 { "123", Tokenizer::TYPE_INTEGER },
212 { "0xab6", Tokenizer::TYPE_INTEGER },
213 { "0XAB6", Tokenizer::TYPE_INTEGER },
214 { "0X1234567", Tokenizer::TYPE_INTEGER },
215 { "0x89abcdef", Tokenizer::TYPE_INTEGER },
216 { "0x89ABCDEF", Tokenizer::TYPE_INTEGER },
217 { "01234567", Tokenizer
[all...]
H A Dtokenizer.h52 class Tokenizer;
83 class LIBPROTOBUF_EXPORT Tokenizer { class in namespace:google::protobuf::io
85 // Construct a Tokenizer that reads and tokenizes text from the given
88 Tokenizer(ZeroCopyInputStream* input, ErrorCollector* error_collector);
89 ~Tokenizer();
192 // comes from a TYPE_FLOAT token parsed by Tokenizer. If it doesn't, the
197 // comes from a TYPE_STRING token parsed by Tokenizer. If it doesn't, the
207 // parsed by a Tokenizer, the result is undefined (possibly an assert
249 GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Tokenizer);
385 inline const Tokenizer
[all...]
/external/parameter-framework/upstream/utility/
H A DTokenizer.h37 /** Tokenizer class
40 * of delimiters (@see Tokenizer::defaultDelimiters).
42 class Tokenizer : private utility::NonCopyable class in inherits:utility::NonCopyable
45 /** Constructs a Tokenizer
54 Tokenizer(const std::string &input, const std::string &delimiters = defaultDelimiters,
56 ~Tokenizer(){};
H A DTokenizer.cpp30 #include "Tokenizer.h"
35 const string Tokenizer::defaultDelimiters = " \n\r\t\v\f";
37 Tokenizer::Tokenizer(const string &input, const string &delimiters, bool mergeDelimiters) function in class:Tokenizer
42 vector<string> Tokenizer::split()
/external/parameter-framework/upstream/test/tokenizer/
H A DTest.cpp31 #include "Tokenizer.h"
44 SCENARIO("Tokenizer tests")
49 Tokenizer tokenizer("a bcd ef");
58 Tokenizer tokenizer("");
67 Tokenizer tokenizer(" a \n\t bc ");
77 Tokenizer tokenizer("/a/bcd/ef g/h/", "/");
88 Tokenizer tokenizer("", Tokenizer::defaultDelimiters, false);
97 Tokenizer tokenizer(",", ",", false);
106 Tokenizer tokenize
[all...]
/external/v8/tools/gyp/tools/Xcode/Specifications/
H A Dgyp.xclangspec73 Tokenizer = "xcode.lang.gyp.lexer.toplevel";
108 Tokenizer = "xcode.lang.gyp.lexer";
121 Tokenizer = "xcode.lang.gyp.lexer";
133 Tokenizer = "xcode.lang.gyp.lexer";
144 Tokenizer = "xcode.lang.gyp.lexer";
155 Tokenizer = "xcode.lang.gyp.lexer";
168 Tokenizer = "xcode.lang.gyp.lexer";
183 Tokenizer = "xcode.lang.gyp.lexer";
/external/chromium-trace/catapult/third_party/vinn/third_party/parse5/lib/simple_api/
H A Dsimple_api_parser.js3 var Tokenizer = require('../tokenization/tokenizer'), variable
55 if (token.type === Tokenizer.CHARACTER_TOKEN ||
56 token.type === Tokenizer.WHITESPACE_CHARACTER_TOKEN ||
57 token.type === Tokenizer.NULL_CHARACTER_TOKEN) {
74 } while (token.type !== Tokenizer.EOF_TOKEN);
82 if (token.type === Tokenizer.START_TAG_TOKEN)
85 else if (token.type === Tokenizer.END_TAG_TOKEN)
88 else if (token.type === Tokenizer.COMMENT_TOKEN)
91 else if (token.type === Tokenizer.DOCTYPE_TOKEN)
H A Dtokenizer_proxy.js3 var Tokenizer = require('../tokenization/tokenizer'), variable
13 //Tokenizer proxy
14 //NOTE: this proxy simulates adjustment of the Tokenizer which performed by standard parser during tree construction.
16 this.tokenizer = new Tokenizer(html, options);
28 if (token.type === Tokenizer.START_TAG_TOKEN)
31 else if (token.type === Tokenizer.END_TAG_TOKEN)
34 else if (token.type === Tokenizer.NULL_CHARACTER_TOKEN && this.inForeignContent) {
35 token.type = Tokenizer.CHARACTER_TOKEN;
64 this.tokenizer.state = Tokenizer.MODE.RCDATA;
67 this.tokenizer.state = Tokenizer
[all...]
/external/deqp/executor/
H A DxeXMLParser.cpp57 Tokenizer::Tokenizer (void) function in class:xe::xml::Tokenizer
65 Tokenizer::~Tokenizer (void)
69 void Tokenizer::clear (void)
77 void Tokenizer::error (const std::string& what)
82 void Tokenizer::feed (const deUint8* bytes, int numBytes)
98 int Tokenizer::getChar (int offset) const
108 void Tokenizer::advance (void)
338 void Tokenizer
[all...]
H A DxeXMLParser.hpp83 class Tokenizer class in namespace:xe::xml
86 Tokenizer (void);
87 ~Tokenizer (void);
103 Tokenizer (const Tokenizer& other);
104 Tokenizer& operator= (const Tokenizer& other);
190 Tokenizer m_tokenizer;
203 inline void Tokenizer::getTokenStr (std::string& dst) const
211 inline void Tokenizer
[all...]
/external/clang/lib/ASTMatchers/Dynamic/
H A DParser.cpp294 const TokenInfo NameToken = Tokenizer->consumeNextToken();
296 if (Tokenizer->nextTokenKind() != TokenInfo::TK_OpenParen) {
306 if ((Tokenizer->nextTokenKind() == TokenInfo::TK_Comma ||
307 Tokenizer->nextTokenKind() == TokenInfo::TK_CloseParen ||
308 Tokenizer->nextTokenKind() == TokenInfo::TK_Eof) &&
328 const TokenInfo OpenToken = Tokenizer->consumeNextToken();
349 while (Tokenizer->nextTokenKind() != TokenInfo::TK_Eof) {
350 if (Tokenizer->nextTokenKind() == TokenInfo::TK_CloseParen) {
352 EndToken = Tokenizer->consumeNextToken();
357 const TokenInfo CommaToken = Tokenizer
524 Parser(CodeTokenizer *Tokenizer, Sema *S, const NamedValueMap *NamedValues, Diagnostics *Error) argument
[all...]
/external/chromium-trace/catapult/third_party/vinn/third_party/parse5/test/fixtures/
H A Dtokenizer_test.js4 Tokenizer = require('../../lib/tokenization/tokenizer');
7 var tokenizer = new Tokenizer(html),
21 case Tokenizer.CHARACTER_TOKEN:
22 case Tokenizer.NULL_CHARACTER_TOKEN:
23 case Tokenizer.WHITESPACE_CHARACTER_TOKEN:
27 case Tokenizer.START_TAG_TOKEN:
46 case Tokenizer.END_TAG_TOKEN:
50 case Tokenizer.COMMENT_TOKEN:
54 case Tokenizer.DOCTYPE_TOKEN:
64 } while (nextToken.type !== Tokenizer
[all...]
/external/ImageMagick/MagickCore/
H A Dtoken.h32 Tokenizer(TokenInfo *,const unsigned int,char *,const size_t,const char *,
/external/libedit/src/
H A Dhistedit.h234 typedef struct tokenizer Tokenizer; typedef in typeref:struct:tokenizer
239 Tokenizer *tok_init(const char *);
240 void tok_end(Tokenizer *);
241 void tok_reset(Tokenizer *);
242 int tok_line(Tokenizer *, const LineInfo *,
244 int tok_str(Tokenizer *, const char *,
H A Dtokenizer.c86 private void FUN(tok,finish)(TYPE(Tokenizer) *);
93 FUN(tok,finish)(TYPE(Tokenizer) *tok)
109 public TYPE(Tokenizer) *
112 TYPE(Tokenizer) *tok = tok_malloc(sizeof(*tok));
151 FUN(tok,reset)(TYPE(Tokenizer) *tok)
166 FUN(tok,end)(TYPE(Tokenizer) *tok)
195 FUN(tok,line)(TYPE(Tokenizer) *tok, const TYPE(LineInfo) *line,
444 FUN(tok,str)(TYPE(Tokenizer) *tok, const Char *line, int *argc,
/external/chromium-trace/catapult/third_party/vinn/third_party/parse5/lib/tree_construction/
H A Dparser.js3 var Tokenizer = require('../tokenization/tokenizer'), variable
88 _[INITIAL_MODE][Tokenizer.CHARACTER_TOKEN] =
89 _[INITIAL_MODE][Tokenizer.NULL_CHARACTER_TOKEN] = tokenInInitialMode;
90 _[INITIAL_MODE][Tokenizer.WHITESPACE_CHARACTER_TOKEN] = ignoreToken;
91 _[INITIAL_MODE][Tokenizer.COMMENT_TOKEN] = appendComment;
92 _[INITIAL_MODE][Tokenizer.DOCTYPE_TOKEN] = doctypeInInitialMode;
93 _[INITIAL_MODE][Tokenizer.START_TAG_TOKEN] =
94 _[INITIAL_MODE][Tokenizer.END_TAG_TOKEN] =
95 _[INITIAL_MODE][Tokenizer.EOF_TOKEN] = tokenInInitialMode;
98 _[BEFORE_HTML_MODE][Tokenizer
[all...]
/external/icu/android_icu4j/src/main/tests/android/icu/dev/test/util/
H A DTestBNF.java15 import android.icu.dev.util.Tokenizer;
228 Tokenizer t = new Tokenizer();
237 while (type != Tokenizer.DONE) {
/external/icu/icu4j/main/tests/translit/src/com/ibm/icu/dev/test/util/
H A DTestBNF.java14 import com.ibm.icu.dev.util.Tokenizer;
227 Tokenizer t = new Tokenizer();
236 while (type != Tokenizer.DONE) {
/external/chromium-trace/catapult/third_party/vinn/third_party/parse5/lib/tokenization/
H A Dtokenizer.js144 //Tokenizer
145 var Tokenizer = module.exports = function (html, options) {
176 Tokenizer.CHARACTER_TOKEN = 'CHARACTER_TOKEN';
177 Tokenizer.NULL_CHARACTER_TOKEN = 'NULL_CHARACTER_TOKEN';
178 Tokenizer.WHITESPACE_CHARACTER_TOKEN = 'WHITESPACE_CHARACTER_TOKEN';
179 Tokenizer.START_TAG_TOKEN = 'START_TAG_TOKEN';
180 Tokenizer.END_TAG_TOKEN = 'END_TAG_TOKEN';
181 Tokenizer.COMMENT_TOKEN = 'COMMENT_TOKEN';
182 Tokenizer.DOCTYPE_TOKEN = 'DOCTYPE_TOKEN';
183 Tokenizer
202 Tokenizer.prototype.getNextToken = function () { class
[all...]
/external/icu/android_icu4j/src/main/tests/android/icu/dev/util/
H A DBNF.java25 private Tokenizer t;
65 t = new Tokenizer();
148 if (type == Tokenizer.DONE) return false;
149 if (type != Tokenizer.STRING) error("missing weight");
200 if (t.next() != Tokenizer.NUMBER) error("missing number");
207 if (type == Tokenizer.NUMBER) {
222 if (token == Tokenizer.STRING) {
227 if (token == Tokenizer.UNICODESET) {
297 if (token != Tokenizer.NUMBER) {
H A DTokenizer.java22 public class Tokenizer { class
71 public Tokenizer setSource(String source) {
77 public Tokenizer setIndex(int index) {
/external/icu/icu4j/main/tests/translit/src/com/ibm/icu/dev/util/
H A DBNF.java24 private Tokenizer t;
64 t = new Tokenizer();
147 if (type == Tokenizer.DONE) return false;
148 if (type != Tokenizer.STRING) error("missing weight");
199 if (t.next() != Tokenizer.NUMBER) error("missing number");
206 if (type == Tokenizer.NUMBER) {
221 if (token == Tokenizer.STRING) {
226 if (token == Tokenizer.UNICODESET) {
296 if (token != Tokenizer.NUMBER) {
H A DTokenizer.java21 public class Tokenizer { class
70 public Tokenizer setSource(String source) {
76 public Tokenizer setIndex(int index) {
/external/protobuf/src/google/protobuf/compiler/
H A Dparser.cc114 inline bool Parser::LookingAtType(io::Tokenizer::TokenType token_type) {
119 return LookingAtType(io::Tokenizer::TYPE_END);
150 if (LookingAtType(io::Tokenizer::TYPE_IDENTIFIER)) {
161 if (LookingAtType(io::Tokenizer::TYPE_INTEGER)) {
163 if (!io::Tokenizer::ParseInteger(input_->current().text,
193 if (LookingAtType(io::Tokenizer::TYPE_INTEGER)) {
194 if (!io::Tokenizer::ParseInteger(input_->current().text, max_value,
209 if (LookingAtType(io::Tokenizer::TYPE_FLOAT)) {
210 *output = io::Tokenizer::ParseFloat(input_->current().text);
213 } else if (LookingAtType(io::Tokenizer
[all...]
/external/protobuf/src/google/protobuf/
H A Dtext_format.cc233 tokenizer_.set_comment_style(io::Tokenizer::SH_COMMENT_STYLE);
252 if (LookingAtType(io::Tokenizer::TYPE_END)) {
267 return suc && LookingAtType(io::Tokenizer::TYPE_END);
656 if (LookingAtType(io::Tokenizer::TYPE_INTEGER)) {
681 if (LookingAtType(io::Tokenizer::TYPE_IDENTIFIER)) {
687 LookingAtType(io::Tokenizer::TYPE_INTEGER)) {
725 if (LookingAtType(io::Tokenizer::TYPE_STRING)) {
726 while (LookingAtType(io::Tokenizer::TYPE_STRING)) {
753 if (!LookingAtType(io::Tokenizer::TYPE_INTEGER) &&
754 !LookingAtType(io::Tokenizer
[all...]

Completed in 3806 milliseconds

1234