idl_parser.cpp revision 641b397f8b79701b44184a52b5b9c6da98eb7580
1/* 2 * Copyright 2014 Google Inc. All rights reserved. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include <algorithm> 18#include <list> 19 20#ifdef _WIN32 21#if !defined(_USE_MATH_DEFINES) 22#define _USE_MATH_DEFINES // For M_PI. 23#endif // !defined(_USE_MATH_DEFINES) 24#endif // _WIN32 25 26#include <math.h> 27 28#include "flatbuffers/idl.h" 29#include "flatbuffers/util.h" 30 31namespace flatbuffers { 32 33const char *const kTypeNames[] = { 34 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \ 35 IDLTYPE, 36 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) 37 #undef FLATBUFFERS_TD 38 nullptr 39}; 40 41const char kTypeSizes[] = { 42 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \ 43 sizeof(CTYPE), 44 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) 45 #undef FLATBUFFERS_TD 46}; 47 48// The enums in the reflection schema should match the ones we use internally. 49// Compare the last element to check if these go out of sync. 50static_assert(BASE_TYPE_UNION == 51 static_cast<BaseType>(reflection::Union), 52 "enums don't match"); 53 54// Any parsing calls have to be wrapped in this macro, which automates 55// handling of recursive error checking a bit. It will check the received 56// CheckedError object, and return straight away on error. 57#define ECHECK(call) { auto ce = (call); if (ce.Check()) return ce; } 58 59// These two functions are called hundreds of times below, so define a short 60// form: 61#define NEXT() ECHECK(Next()) 62#define EXPECT(tok) ECHECK(Expect(tok)) 63 64static bool ValidateUTF8(const std::string &str) { 65 const char *s = &str[0]; 66 const char * const sEnd = s + str.length(); 67 while (s < sEnd) { 68 if (FromUTF8(&s) < 0) { 69 return false; 70 } 71 } 72 return true; 73} 74 75CheckedError Parser::Error(const std::string &msg) { 76 error_ = file_being_parsed_.length() ? AbsolutePath(file_being_parsed_) : ""; 77 #ifdef _WIN32 78 error_ += "(" + NumToString(line_) + ")"; // MSVC alike 79 #else 80 if (file_being_parsed_.length()) error_ += ":"; 81 error_ += NumToString(line_) + ":0"; // gcc alike 82 #endif 83 error_ += ": error: " + msg; 84 return CheckedError(true); 85} 86 87inline CheckedError NoError() { return CheckedError(false); } 88 89// Ensure that integer values we parse fit inside the declared integer type. 90CheckedError Parser::CheckBitsFit(int64_t val, size_t bits) { 91 // Left-shifting a 64-bit value by 64 bits or more is undefined 92 // behavior (C99 6.5.7), so check *before* we shift. 93 if (bits < 64) { 94 // Bits we allow to be used. 95 auto mask = static_cast<int64_t>((1ull << bits) - 1); 96 if ((val & ~mask) != 0 && // Positive or unsigned. 97 (val | mask) != -1) // Negative. 98 return Error("constant does not fit in a " + NumToString(bits) + 99 "-bit field"); 100 } 101 return NoError(); 102} 103 104// atot: templated version of atoi/atof: convert a string to an instance of T. 105template<typename T> inline CheckedError atot(const char *s, Parser &parser, 106 T *val) { 107 int64_t i = StringToInt(s); 108 ECHECK(parser.CheckBitsFit(i, sizeof(T) * 8)); 109 *val = (T)i; 110 return NoError(); 111} 112template<> inline CheckedError atot<bool>(const char *s, Parser &parser, 113 bool *val) { 114 (void)parser; 115 *val = 0 != atoi(s); 116 return NoError(); 117} 118template<> inline CheckedError atot<float>(const char *s, Parser &parser, 119 float *val) { 120 (void)parser; 121 *val = static_cast<float>(strtod(s, nullptr)); 122 return NoError(); 123} 124template<> inline CheckedError atot<double>(const char *s, Parser &parser, 125 double *val) { 126 (void)parser; 127 *val = strtod(s, nullptr); 128 return NoError(); 129} 130 131template<> inline CheckedError atot<Offset<void>>(const char *s, Parser &parser, 132 Offset<void> *val) { 133 (void)parser; 134 *val = Offset<void>(atoi(s)); 135 return NoError(); 136} 137 138std::string Namespace::GetFullyQualifiedName(const std::string &name, 139 size_t max_components) const { 140 // Early exit if we don't have a defined namespace. 141 if (components.size() == 0 || !max_components) { 142 return name; 143 } 144 std::stringstream stream; 145 for (size_t i = 0; i < std::min(components.size(), max_components); 146 i++) { 147 if (i) { 148 stream << "."; 149 } 150 stream << components[i]; 151 } 152 153 stream << "." << name; 154 return stream.str(); 155} 156 157 158 159// Declare tokens we'll use. Single character tokens are represented by their 160// ascii character code (e.g. '{'), others above 256. 161#define FLATBUFFERS_GEN_TOKENS(TD) \ 162 TD(Eof, 256, "end of file") \ 163 TD(StringConstant, 257, "string constant") \ 164 TD(IntegerConstant, 258, "integer constant") \ 165 TD(FloatConstant, 259, "float constant") \ 166 TD(Identifier, 260, "identifier") \ 167 TD(Table, 261, "table") \ 168 TD(Struct, 262, "struct") \ 169 TD(Enum, 263, "enum") \ 170 TD(Union, 264, "union") \ 171 TD(NameSpace, 265, "namespace") \ 172 TD(RootType, 266, "root_type") \ 173 TD(FileIdentifier, 267, "file_identifier") \ 174 TD(FileExtension, 268, "file_extension") \ 175 TD(Include, 269, "include") \ 176 TD(Attribute, 270, "attribute") \ 177 TD(Null, 271, "null") \ 178 TD(Service, 272, "rpc_service") 179#ifdef __GNUC__ 180__extension__ // Stop GCC complaining about trailing comma with -Wpendantic. 181#endif 182enum { 183 #define FLATBUFFERS_TOKEN(NAME, VALUE, STRING) kToken ## NAME = VALUE, 184 FLATBUFFERS_GEN_TOKENS(FLATBUFFERS_TOKEN) 185 #undef FLATBUFFERS_TOKEN 186 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \ 187 kToken ## ENUM, 188 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) 189 #undef FLATBUFFERS_TD 190}; 191 192static std::string TokenToString(int t) { 193 static const char *tokens[] = { 194 #define FLATBUFFERS_TOKEN(NAME, VALUE, STRING) STRING, 195 FLATBUFFERS_GEN_TOKENS(FLATBUFFERS_TOKEN) 196 #undef FLATBUFFERS_TOKEN 197 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \ 198 IDLTYPE, 199 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) 200 #undef FLATBUFFERS_TD 201 }; 202 if (t < 256) { // A single ascii char token. 203 std::string s; 204 s.append(1, static_cast<char>(t)); 205 return s; 206 } else { // Other tokens. 207 return tokens[t - 256]; 208 } 209} 210 211std::string Parser::TokenToStringId(int t) { 212 return TokenToString(t) + (t == kTokenIdentifier ? ": " + attribute_ : ""); 213} 214 215// Parses exactly nibbles worth of hex digits into a number, or error. 216CheckedError Parser::ParseHexNum(int nibbles, int64_t *val) { 217 for (int i = 0; i < nibbles; i++) 218 if (!isxdigit(static_cast<const unsigned char>(cursor_[i]))) 219 return Error("escape code must be followed by " + NumToString(nibbles) + 220 " hex digits"); 221 std::string target(cursor_, cursor_ + nibbles); 222 *val = StringToUInt(target.c_str(), nullptr, 16); 223 cursor_ += nibbles; 224 return NoError(); 225} 226 227CheckedError Parser::SkipByteOrderMark() { 228 if (static_cast<unsigned char>(*cursor_) != 0xef) return NoError(); 229 cursor_++; 230 if (static_cast<unsigned char>(*cursor_) != 0xbb) return Error("invalid utf-8 byte order mark"); 231 cursor_++; 232 if (static_cast<unsigned char>(*cursor_) != 0xbf) return Error("invalid utf-8 byte order mark"); 233 cursor_++; 234 return NoError(); 235} 236 237bool IsIdentifierStart(char c) { 238 return isalpha(static_cast<unsigned char>(c)) || c == '_'; 239} 240 241CheckedError Parser::Next() { 242 doc_comment_.clear(); 243 bool seen_newline = false; 244 attribute_.clear(); 245 for (;;) { 246 char c = *cursor_++; 247 token_ = c; 248 switch (c) { 249 case '\0': cursor_--; token_ = kTokenEof; return NoError(); 250 case ' ': case '\r': case '\t': break; 251 case '\n': line_++; seen_newline = true; break; 252 case '{': case '}': case '(': case ')': case '[': case ']': 253 case ',': case ':': case ';': case '=': return NoError(); 254 case '.': 255 if(!isdigit(static_cast<const unsigned char>(*cursor_))) return NoError(); 256 return Error("floating point constant can\'t start with \".\""); 257 case '\"': 258 case '\'': { 259 int unicode_high_surrogate = -1; 260 261 while (*cursor_ != c) { 262 if (*cursor_ < ' ' && *cursor_ >= 0) 263 return Error("illegal character in string constant"); 264 if (*cursor_ == '\\') { 265 cursor_++; 266 if (unicode_high_surrogate != -1 && 267 *cursor_ != 'u') { 268 return Error( 269 "illegal Unicode sequence (unpaired high surrogate)"); 270 } 271 switch (*cursor_) { 272 case 'n': attribute_ += '\n'; cursor_++; break; 273 case 't': attribute_ += '\t'; cursor_++; break; 274 case 'r': attribute_ += '\r'; cursor_++; break; 275 case 'b': attribute_ += '\b'; cursor_++; break; 276 case 'f': attribute_ += '\f'; cursor_++; break; 277 case '\"': attribute_ += '\"'; cursor_++; break; 278 case '\'': attribute_ += '\''; cursor_++; break; 279 case '\\': attribute_ += '\\'; cursor_++; break; 280 case '/': attribute_ += '/'; cursor_++; break; 281 case 'x': { // Not in the JSON standard 282 cursor_++; 283 int64_t val; 284 ECHECK(ParseHexNum(2, &val)); 285 attribute_ += static_cast<char>(val); 286 break; 287 } 288 case 'u': { 289 cursor_++; 290 int64_t val; 291 ECHECK(ParseHexNum(4, &val)); 292 if (val >= 0xD800 && val <= 0xDBFF) { 293 if (unicode_high_surrogate != -1) { 294 return Error( 295 "illegal Unicode sequence (multiple high surrogates)"); 296 } else { 297 unicode_high_surrogate = static_cast<int>(val); 298 } 299 } else if (val >= 0xDC00 && val <= 0xDFFF) { 300 if (unicode_high_surrogate == -1) { 301 return Error( 302 "illegal Unicode sequence (unpaired low surrogate)"); 303 } else { 304 int code_point = 0x10000 + 305 ((unicode_high_surrogate & 0x03FF) << 10) + 306 (val & 0x03FF); 307 ToUTF8(code_point, &attribute_); 308 unicode_high_surrogate = -1; 309 } 310 } else { 311 if (unicode_high_surrogate != -1) { 312 return Error( 313 "illegal Unicode sequence (unpaired high surrogate)"); 314 } 315 ToUTF8(static_cast<int>(val), &attribute_); 316 } 317 break; 318 } 319 default: return Error("unknown escape code in string constant"); 320 } 321 } else { // printable chars + UTF-8 bytes 322 if (unicode_high_surrogate != -1) { 323 return Error( 324 "illegal Unicode sequence (unpaired high surrogate)"); 325 } 326 attribute_ += *cursor_++; 327 } 328 } 329 if (unicode_high_surrogate != -1) { 330 return Error( 331 "illegal Unicode sequence (unpaired high surrogate)"); 332 } 333 cursor_++; 334 if (!opts.allow_non_utf8 && !ValidateUTF8(attribute_)) { 335 return Error("illegal UTF-8 sequence"); 336 } 337 token_ = kTokenStringConstant; 338 return NoError(); 339 } 340 case '/': 341 if (*cursor_ == '/') { 342 const char *start = ++cursor_; 343 while (*cursor_ && *cursor_ != '\n' && *cursor_ != '\r') cursor_++; 344 if (*start == '/') { // documentation comment 345 if (cursor_ != source_ && !seen_newline) 346 return Error( 347 "a documentation comment should be on a line on its own"); 348 doc_comment_.push_back(std::string(start + 1, cursor_)); 349 } 350 break; 351 } else if (*cursor_ == '*') { 352 cursor_++; 353 // TODO: make nested. 354 while (*cursor_ != '*' || cursor_[1] != '/') { 355 if (*cursor_ == '\n') line_++; 356 if (!*cursor_) return Error("end of file in comment"); 357 cursor_++; 358 } 359 cursor_ += 2; 360 break; 361 } 362 // fall thru 363 default: 364 if (IsIdentifierStart(c)) { 365 // Collect all chars of an identifier: 366 const char *start = cursor_ - 1; 367 while (isalnum(static_cast<unsigned char>(*cursor_)) || 368 *cursor_ == '_') 369 cursor_++; 370 attribute_.append(start, cursor_); 371 // First, see if it is a type keyword from the table of types: 372 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, \ 373 PTYPE) \ 374 if (attribute_ == IDLTYPE) { \ 375 token_ = kToken ## ENUM; \ 376 return NoError(); \ 377 } 378 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) 379 #undef FLATBUFFERS_TD 380 // If it's a boolean constant keyword, turn those into integers, 381 // which simplifies our logic downstream. 382 if (attribute_ == "true" || attribute_ == "false") { 383 attribute_ = NumToString(attribute_ == "true"); 384 token_ = kTokenIntegerConstant; 385 return NoError(); 386 } 387 // Check for declaration keywords: 388 if (attribute_ == "table") { 389 token_ = kTokenTable; 390 return NoError(); 391 } 392 if (attribute_ == "struct") { 393 token_ = kTokenStruct; 394 return NoError(); 395 } 396 if (attribute_ == "enum") { 397 token_ = kTokenEnum; 398 return NoError(); 399 } 400 if (attribute_ == "union") { 401 token_ = kTokenUnion; 402 return NoError(); 403 } 404 if (attribute_ == "namespace") { 405 token_ = kTokenNameSpace; 406 return NoError(); 407 } 408 if (attribute_ == "root_type") { 409 token_ = kTokenRootType; 410 return NoError(); 411 } 412 if (attribute_ == "include") { 413 token_ = kTokenInclude; 414 return NoError(); 415 } 416 if (attribute_ == "attribute") { 417 token_ = kTokenAttribute; 418 return NoError(); 419 } 420 if (attribute_ == "file_identifier") { 421 token_ = kTokenFileIdentifier; 422 return NoError(); 423 } 424 if (attribute_ == "file_extension") { 425 token_ = kTokenFileExtension; 426 return NoError(); 427 } 428 if (attribute_ == "null") { 429 token_ = kTokenNull; 430 return NoError(); 431 } 432 if (attribute_ == "rpc_service") { 433 token_ = kTokenService; 434 return NoError(); 435 } 436 // If not, it is a user-defined identifier: 437 token_ = kTokenIdentifier; 438 return NoError(); 439 } else if (isdigit(static_cast<unsigned char>(c)) || c == '-') { 440 const char *start = cursor_ - 1; 441 if (c == '-' && *cursor_ == '0' && (cursor_[1] == 'x' || cursor_[1] == 'X')) { 442 ++start; 443 ++cursor_; 444 attribute_.append(&c, &c + 1); 445 c = '0'; 446 } 447 if (c == '0' && (*cursor_ == 'x' || *cursor_ == 'X')) { 448 cursor_++; 449 while (isxdigit(static_cast<unsigned char>(*cursor_))) cursor_++; 450 attribute_.append(start + 2, cursor_); 451 attribute_ = NumToString(StringToUInt(attribute_.c_str(), nullptr, 16)); 452 token_ = kTokenIntegerConstant; 453 return NoError(); 454 } 455 while (isdigit(static_cast<unsigned char>(*cursor_))) cursor_++; 456 if (*cursor_ == '.' || *cursor_ == 'e' || *cursor_ == 'E') { 457 if (*cursor_ == '.') { 458 cursor_++; 459 while (isdigit(static_cast<unsigned char>(*cursor_))) cursor_++; 460 } 461 // See if this float has a scientific notation suffix. Both JSON 462 // and C++ (through strtod() we use) have the same format: 463 if (*cursor_ == 'e' || *cursor_ == 'E') { 464 cursor_++; 465 if (*cursor_ == '+' || *cursor_ == '-') cursor_++; 466 while (isdigit(static_cast<unsigned char>(*cursor_))) cursor_++; 467 } 468 token_ = kTokenFloatConstant; 469 } else { 470 token_ = kTokenIntegerConstant; 471 } 472 attribute_.append(start, cursor_); 473 return NoError(); 474 } 475 std::string ch; 476 ch = c; 477 if (c < ' ' || c > '~') ch = "code: " + NumToString(c); 478 return Error("illegal character: " + ch); 479 } 480 } 481} 482 483// Check if a given token is next. 484bool Parser::Is(int t) { 485 return t == token_; 486} 487 488// Expect a given token to be next, consume it, or error if not present. 489CheckedError Parser::Expect(int t) { 490 if (t != token_) { 491 return Error("expecting: " + TokenToString(t) + " instead got: " + 492 TokenToStringId(token_)); 493 } 494 NEXT(); 495 return NoError(); 496} 497 498CheckedError Parser::ParseNamespacing(std::string *id, std::string *last) { 499 while (Is('.')) { 500 NEXT(); 501 *id += "."; 502 *id += attribute_; 503 if (last) *last = attribute_; 504 EXPECT(kTokenIdentifier); 505 } 506 return NoError(); 507} 508 509EnumDef *Parser::LookupEnum(const std::string &id) { 510 // Search thru parent namespaces. 511 for (int components = static_cast<int>(namespaces_.back()->components.size()); 512 components >= 0; components--) { 513 auto ed = enums_.Lookup( 514 namespaces_.back()->GetFullyQualifiedName(id, components)); 515 if (ed) return ed; 516 } 517 return nullptr; 518} 519 520CheckedError Parser::ParseTypeIdent(Type &type) { 521 std::string id = attribute_; 522 EXPECT(kTokenIdentifier); 523 ECHECK(ParseNamespacing(&id, nullptr)); 524 auto enum_def = LookupEnum(id); 525 if (enum_def) { 526 type = enum_def->underlying_type; 527 if (enum_def->is_union) type.base_type = BASE_TYPE_UNION; 528 } else { 529 type.base_type = BASE_TYPE_STRUCT; 530 type.struct_def = LookupCreateStruct(id); 531 } 532 return NoError(); 533} 534 535// Parse any IDL type. 536CheckedError Parser::ParseType(Type &type) { 537 if (token_ >= kTokenBOOL && token_ <= kTokenSTRING) { 538 type.base_type = static_cast<BaseType>(token_ - kTokenNONE); 539 NEXT(); 540 } else { 541 if (token_ == kTokenIdentifier) { 542 ECHECK(ParseTypeIdent(type)); 543 } else if (token_ == '[') { 544 NEXT(); 545 Type subtype; 546 ECHECK(ParseType(subtype)); 547 if (subtype.base_type == BASE_TYPE_VECTOR) { 548 // We could support this, but it will complicate things, and it's 549 // easier to work around with a struct around the inner vector. 550 return Error( 551 "nested vector types not supported (wrap in table first)."); 552 } 553 if (subtype.base_type == BASE_TYPE_UNION) { 554 // We could support this if we stored a struct of 2 elements per 555 // union element. 556 return Error( 557 "vector of union types not supported (wrap in table first)."); 558 } 559 type = Type(BASE_TYPE_VECTOR, subtype.struct_def, subtype.enum_def); 560 type.element = subtype.base_type; 561 EXPECT(']'); 562 } else { 563 return Error("illegal type syntax"); 564 } 565 } 566 return NoError(); 567} 568 569CheckedError Parser::AddField(StructDef &struct_def, const std::string &name, 570 const Type &type, FieldDef **dest) { 571 auto &field = *new FieldDef(); 572 field.value.offset = 573 FieldIndexToOffset(static_cast<voffset_t>(struct_def.fields.vec.size())); 574 field.name = name; 575 field.file = struct_def.file; 576 field.value.type = type; 577 if (struct_def.fixed) { // statically compute the field offset 578 auto size = InlineSize(type); 579 auto alignment = InlineAlignment(type); 580 // structs_ need to have a predictable format, so we need to align to 581 // the largest scalar 582 struct_def.minalign = std::max(struct_def.minalign, alignment); 583 struct_def.PadLastField(alignment); 584 field.value.offset = static_cast<voffset_t>(struct_def.bytesize); 585 struct_def.bytesize += size; 586 } 587 if (struct_def.fields.Add(name, &field)) 588 return Error("field already exists: " + name); 589 *dest = &field; 590 return NoError(); 591} 592 593CheckedError Parser::ParseField(StructDef &struct_def) { 594 std::string name = attribute_; 595 std::vector<std::string> dc = doc_comment_; 596 EXPECT(kTokenIdentifier); 597 EXPECT(':'); 598 Type type; 599 ECHECK(ParseType(type)); 600 601 if (struct_def.fixed && !IsScalar(type.base_type) && !IsStruct(type)) 602 return Error("structs_ may contain only scalar or struct fields"); 603 604 FieldDef *typefield = nullptr; 605 if (type.base_type == BASE_TYPE_UNION) { 606 // For union fields, add a second auto-generated field to hold the type, 607 // with a special suffix. 608 ECHECK(AddField(struct_def, name + UnionTypeFieldSuffix(), 609 type.enum_def->underlying_type, &typefield)); 610 } 611 612 FieldDef *field; 613 ECHECK(AddField(struct_def, name, type, &field)); 614 615 if (token_ == '=') { 616 NEXT(); 617 if (!IsScalar(type.base_type)) 618 return Error("default values currently only supported for scalars"); 619 ECHECK(ParseSingleValue(field->value)); 620 } 621 if (IsFloat(field->value.type.base_type)) { 622 if (!strpbrk(field->value.constant.c_str(), ".eE")) 623 field->value.constant += ".0"; 624 } 625 626 if (type.enum_def && 627 IsScalar(type.base_type) && 628 !struct_def.fixed && 629 !type.enum_def->attributes.Lookup("bit_flags") && 630 !type.enum_def->ReverseLookup(static_cast<int>( 631 StringToInt(field->value.constant.c_str())))) 632 return Error("enum " + type.enum_def->name + 633 " does not have a declaration for this field\'s default of " + 634 field->value.constant); 635 636 field->doc_comment = dc; 637 ECHECK(ParseMetaData(&field->attributes)); 638 field->deprecated = field->attributes.Lookup("deprecated") != nullptr; 639 auto hash_name = field->attributes.Lookup("hash"); 640 if (hash_name) { 641 switch (type.base_type) { 642 case BASE_TYPE_INT: 643 case BASE_TYPE_UINT: { 644 if (FindHashFunction32(hash_name->constant.c_str()) == nullptr) 645 return Error("Unknown hashing algorithm for 32 bit types: " + 646 hash_name->constant); 647 break; 648 } 649 case BASE_TYPE_LONG: 650 case BASE_TYPE_ULONG: { 651 if (FindHashFunction64(hash_name->constant.c_str()) == nullptr) 652 return Error("Unknown hashing algorithm for 64 bit types: " + 653 hash_name->constant); 654 break; 655 } 656 default: 657 return Error( 658 "only int, uint, long and ulong data types support hashing."); 659 } 660 } 661 auto cpp_type = field->attributes.Lookup("cpp_type"); 662 if (cpp_type) { 663 if (!hash_name) 664 return Error("cpp_type can only be used with a hashed field"); 665 } 666 if (field->deprecated && struct_def.fixed) 667 return Error("can't deprecate fields in a struct"); 668 field->required = field->attributes.Lookup("required") != nullptr; 669 if (field->required && (struct_def.fixed || 670 IsScalar(field->value.type.base_type))) 671 return Error("only non-scalar fields in tables may be 'required'"); 672 field->key = field->attributes.Lookup("key") != nullptr; 673 if (field->key) { 674 if (struct_def.has_key) 675 return Error("only one field may be set as 'key'"); 676 struct_def.has_key = true; 677 if (!IsScalar(field->value.type.base_type)) { 678 field->required = true; 679 if (field->value.type.base_type != BASE_TYPE_STRING) 680 return Error("'key' field must be string or scalar type"); 681 } 682 } 683 684 field->native_inline = field->attributes.Lookup("native_inline") != nullptr; 685 if (field->native_inline && !IsStruct(field->value.type)) 686 return Error("native_inline can only be defined on structs'"); 687 688 auto nested = field->attributes.Lookup("nested_flatbuffer"); 689 if (nested) { 690 if (nested->type.base_type != BASE_TYPE_STRING) 691 return Error( 692 "nested_flatbuffer attribute must be a string (the root type)"); 693 if (field->value.type.base_type != BASE_TYPE_VECTOR || 694 field->value.type.element != BASE_TYPE_UCHAR) 695 return Error( 696 "nested_flatbuffer attribute may only apply to a vector of ubyte"); 697 // This will cause an error if the root type of the nested flatbuffer 698 // wasn't defined elsewhere. 699 LookupCreateStruct(nested->constant); 700 } 701 702 if (typefield) { 703 // If this field is a union, and it has a manually assigned id, 704 // the automatically added type field should have an id as well (of N - 1). 705 auto attr = field->attributes.Lookup("id"); 706 if (attr) { 707 auto id = atoi(attr->constant.c_str()); 708 auto val = new Value(); 709 val->type = attr->type; 710 val->constant = NumToString(id - 1); 711 typefield->attributes.Add("id", val); 712 } 713 } 714 715 EXPECT(';'); 716 return NoError(); 717} 718 719CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field, 720 size_t parent_fieldn, 721 const StructDef *parent_struct_def) { 722 switch (val.type.base_type) { 723 case BASE_TYPE_UNION: { 724 assert(field); 725 std::string constant; 726 if (!parent_fieldn || 727 field_stack_.back().second->value.type.base_type != BASE_TYPE_UTYPE) { 728 // We haven't seen the type field yet. Sadly a lot of JSON writers 729 // output these in alphabetical order, meaning it comes after this 730 // value. So we scan past the value to find it, then come back here. 731 auto type_name = field->name + UnionTypeFieldSuffix(); 732 assert(parent_struct_def); 733 auto type_field = parent_struct_def->fields.Lookup(type_name); 734 assert(type_field); // Guaranteed by ParseField(). 735 // Remember where we are in the source file, so we can come back here. 736 auto backup = *static_cast<ParserState *>(this); 737 ECHECK(SkipAnyJsonValue()); // The table. 738 EXPECT(','); 739 auto next_name = attribute_; 740 if (Is(kTokenStringConstant)) { 741 NEXT(); 742 } else { 743 EXPECT(kTokenIdentifier); 744 } 745 if (next_name != type_name) 746 return Error("missing type field after this union value: " + 747 type_name); 748 EXPECT(':'); 749 Value type_val = type_field->value; 750 ECHECK(ParseAnyValue(type_val, type_field, 0, nullptr)); 751 constant = type_val.constant; 752 // Got the information we needed, now rewind: 753 *static_cast<ParserState *>(this) = backup; 754 } else { 755 constant = field_stack_.back().first.constant; 756 } 757 uint8_t enum_idx; 758 ECHECK(atot(constant.c_str(), *this, &enum_idx)); 759 auto enum_val = val.type.enum_def->ReverseLookup(enum_idx); 760 if (!enum_val) return Error("illegal type id for: " + field->name); 761 ECHECK(ParseTable(*enum_val->struct_def, &val.constant, nullptr)); 762 break; 763 } 764 case BASE_TYPE_STRUCT: 765 ECHECK(ParseTable(*val.type.struct_def, &val.constant, nullptr)); 766 break; 767 case BASE_TYPE_STRING: { 768 auto s = attribute_; 769 EXPECT(kTokenStringConstant); 770 val.constant = NumToString(builder_.CreateString(s).o); 771 break; 772 } 773 case BASE_TYPE_VECTOR: { 774 EXPECT('['); 775 uoffset_t off; 776 ECHECK(ParseVector(val.type.VectorType(), &off)); 777 val.constant = NumToString(off); 778 break; 779 } 780 case BASE_TYPE_INT: 781 case BASE_TYPE_UINT: 782 case BASE_TYPE_LONG: 783 case BASE_TYPE_ULONG: { 784 if (field && field->attributes.Lookup("hash") && 785 (token_ == kTokenIdentifier || token_ == kTokenStringConstant)) { 786 ECHECK(ParseHash(val, field)); 787 } else { 788 ECHECK(ParseSingleValue(val)); 789 } 790 break; 791 } 792 default: 793 ECHECK(ParseSingleValue(val)); 794 break; 795 } 796 return NoError(); 797} 798 799void Parser::SerializeStruct(const StructDef &struct_def, const Value &val) { 800 assert(val.constant.length() == struct_def.bytesize); 801 builder_.Align(struct_def.minalign); 802 builder_.PushBytes(reinterpret_cast<const uint8_t *>(val.constant.c_str()), 803 struct_def.bytesize); 804 builder_.AddStructOffset(val.offset, builder_.GetSize()); 805} 806 807CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, 808 uoffset_t *ovalue) { 809 EXPECT('{'); 810 size_t fieldn = 0; 811 for (;;) { 812 if ((!opts.strict_json || !fieldn) && Is('}')) { NEXT(); break; } 813 std::string name = attribute_; 814 if (Is(kTokenStringConstant)) { 815 NEXT(); 816 } else { 817 EXPECT(opts.strict_json ? kTokenStringConstant : kTokenIdentifier); 818 } 819 auto field = struct_def.fields.Lookup(name); 820 if (!field) { 821 if (!opts.skip_unexpected_fields_in_json) { 822 return Error("unknown field: " + name); 823 } else { 824 EXPECT(':'); 825 ECHECK(SkipAnyJsonValue()); 826 } 827 } else { 828 EXPECT(':'); 829 if (Is(kTokenNull)) { 830 NEXT(); // Ignore this field. 831 } else { 832 Value val = field->value; 833 ECHECK(ParseAnyValue(val, field, fieldn, &struct_def)); 834 size_t i = field_stack_.size(); 835 // Hardcoded insertion-sort with error-check. 836 // If fields are specified in order, then this loop exits immediately. 837 for (; i > field_stack_.size() - fieldn; i--) { 838 auto existing_field = field_stack_[i - 1].second; 839 if (existing_field == field) 840 return Error("field set more than once: " + field->name); 841 if (existing_field->value.offset < field->value.offset) break; 842 } 843 field_stack_.insert(field_stack_.begin() + i, std::make_pair(val, field)); 844 fieldn++; 845 } 846 } 847 if (Is('}')) { NEXT(); break; } 848 EXPECT(','); 849 } 850 851 if (struct_def.fixed && fieldn != struct_def.fields.vec.size()) 852 return Error("struct: wrong number of initializers: " + struct_def.name); 853 854 auto start = struct_def.fixed 855 ? builder_.StartStruct(struct_def.minalign) 856 : builder_.StartTable(); 857 858 for (size_t size = struct_def.sortbysize ? sizeof(largest_scalar_t) : 1; 859 size; 860 size /= 2) { 861 // Go through elements in reverse, since we're building the data backwards. 862 for (auto it = field_stack_.rbegin(); 863 it != field_stack_.rbegin() + fieldn; ++it) { 864 auto &field_value = it->first; 865 auto field = it->second; 866 if (!struct_def.sortbysize || 867 size == SizeOf(field_value.type.base_type)) { 868 switch (field_value.type.base_type) { 869 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, \ 870 PTYPE) \ 871 case BASE_TYPE_ ## ENUM: \ 872 builder_.Pad(field->padding); \ 873 if (struct_def.fixed) { \ 874 CTYPE val; \ 875 ECHECK(atot(field_value.constant.c_str(), *this, &val)); \ 876 builder_.PushElement(val); \ 877 } else { \ 878 CTYPE val, valdef; \ 879 ECHECK(atot(field_value.constant.c_str(), *this, &val)); \ 880 ECHECK(atot(field->value.constant.c_str(), *this, &valdef)); \ 881 builder_.AddElement(field_value.offset, val, valdef); \ 882 } \ 883 break; 884 FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD); 885 #undef FLATBUFFERS_TD 886 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, \ 887 PTYPE) \ 888 case BASE_TYPE_ ## ENUM: \ 889 builder_.Pad(field->padding); \ 890 if (IsStruct(field->value.type)) { \ 891 SerializeStruct(*field->value.type.struct_def, field_value); \ 892 } else { \ 893 CTYPE val; \ 894 ECHECK(atot(field_value.constant.c_str(), *this, &val)); \ 895 builder_.AddOffset(field_value.offset, val); \ 896 } \ 897 break; 898 FLATBUFFERS_GEN_TYPES_POINTER(FLATBUFFERS_TD); 899 #undef FLATBUFFERS_TD 900 } 901 } 902 } 903 } 904 for (size_t i = 0; i < fieldn; i++) field_stack_.pop_back(); 905 906 if (struct_def.fixed) { 907 builder_.ClearOffsets(); 908 builder_.EndStruct(); 909 assert(value); 910 // Temporarily store this struct in the value string, since it is to 911 // be serialized in-place elsewhere. 912 value->assign( 913 reinterpret_cast<const char *>(builder_.GetCurrentBufferPointer()), 914 struct_def.bytesize); 915 builder_.PopBytes(struct_def.bytesize); 916 assert(!ovalue); 917 } else { 918 auto val = builder_.EndTable(start, 919 static_cast<voffset_t>(struct_def.fields.vec.size())); 920 if (ovalue) *ovalue = val; 921 if (value) *value = NumToString(val); 922 } 923 return NoError(); 924} 925 926CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue) { 927 int count = 0; 928 for (;;) { 929 if ((!opts.strict_json || !count) && Is(']')) { NEXT(); break; } 930 Value val; 931 val.type = type; 932 ECHECK(ParseAnyValue(val, nullptr, 0, nullptr)); 933 field_stack_.push_back(std::make_pair(val, nullptr)); 934 count++; 935 if (Is(']')) { NEXT(); break; } 936 EXPECT(','); 937 } 938 939 builder_.StartVector(count * InlineSize(type) / InlineAlignment(type), 940 InlineAlignment(type)); 941 for (int i = 0; i < count; i++) { 942 // start at the back, since we're building the data backwards. 943 auto &val = field_stack_.back().first; 944 switch (val.type.base_type) { 945 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \ 946 case BASE_TYPE_ ## ENUM: \ 947 if (IsStruct(val.type)) SerializeStruct(*val.type.struct_def, val); \ 948 else { \ 949 CTYPE elem; \ 950 ECHECK(atot(val.constant.c_str(), *this, &elem)); \ 951 builder_.PushElement(elem); \ 952 } \ 953 break; 954 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) 955 #undef FLATBUFFERS_TD 956 } 957 field_stack_.pop_back(); 958 } 959 960 builder_.ClearOffsets(); 961 *ovalue = builder_.EndVector(count); 962 return NoError(); 963} 964 965CheckedError Parser::ParseMetaData(SymbolTable<Value> *attributes) { 966 if (Is('(')) { 967 NEXT(); 968 for (;;) { 969 auto name = attribute_; 970 EXPECT(kTokenIdentifier); 971 if (known_attributes_.find(name) == known_attributes_.end()) 972 return Error("user define attributes must be declared before use: " + 973 name); 974 auto e = new Value(); 975 attributes->Add(name, e); 976 if (Is(':')) { 977 NEXT(); 978 ECHECK(ParseSingleValue(*e)); 979 } 980 if (Is(')')) { NEXT(); break; } 981 EXPECT(','); 982 } 983 } 984 return NoError(); 985} 986 987CheckedError Parser::TryTypedValue(int dtoken, bool check, Value &e, 988 BaseType req, bool *destmatch) { 989 bool match = dtoken == token_; 990 if (match) { 991 *destmatch = true; 992 e.constant = attribute_; 993 if (!check) { 994 if (e.type.base_type == BASE_TYPE_NONE) { 995 e.type.base_type = req; 996 } else { 997 return Error(std::string("type mismatch: expecting: ") + 998 kTypeNames[e.type.base_type] + 999 ", found: " + 1000 kTypeNames[req]); 1001 } 1002 } 1003 NEXT(); 1004 } 1005 return NoError(); 1006} 1007 1008CheckedError Parser::ParseEnumFromString(Type &type, int64_t *result) { 1009 *result = 0; 1010 // Parse one or more enum identifiers, separated by spaces. 1011 const char *next = attribute_.c_str(); 1012 do { 1013 const char *divider = strchr(next, ' '); 1014 std::string word; 1015 if (divider) { 1016 word = std::string(next, divider); 1017 next = divider + strspn(divider, " "); 1018 } else { 1019 word = next; 1020 next += word.length(); 1021 } 1022 if (type.enum_def) { // The field has an enum type 1023 auto enum_val = type.enum_def->vals.Lookup(word); 1024 if (!enum_val) 1025 return Error("unknown enum value: " + word + 1026 ", for enum: " + type.enum_def->name); 1027 *result |= enum_val->value; 1028 } else { // No enum type, probably integral field. 1029 if (!IsInteger(type.base_type)) 1030 return Error("not a valid value for this field: " + word); 1031 // TODO: could check if its a valid number constant here. 1032 const char *dot = strrchr(word.c_str(), '.'); 1033 if (!dot) 1034 return Error("enum values need to be qualified by an enum type"); 1035 std::string enum_def_str(word.c_str(), dot); 1036 std::string enum_val_str(dot + 1, word.c_str() + word.length()); 1037 auto enum_def = LookupEnum(enum_def_str); 1038 if (!enum_def) return Error("unknown enum: " + enum_def_str); 1039 auto enum_val = enum_def->vals.Lookup(enum_val_str); 1040 if (!enum_val) return Error("unknown enum value: " + enum_val_str); 1041 *result |= enum_val->value; 1042 } 1043 } while(*next); 1044 return NoError(); 1045} 1046 1047 1048CheckedError Parser::ParseHash(Value &e, FieldDef* field) { 1049 assert(field); 1050 Value *hash_name = field->attributes.Lookup("hash"); 1051 switch (e.type.base_type) { 1052 case BASE_TYPE_INT: 1053 case BASE_TYPE_UINT: { 1054 auto hash = FindHashFunction32(hash_name->constant.c_str()); 1055 uint32_t hashed_value = hash(attribute_.c_str()); 1056 e.constant = NumToString(hashed_value); 1057 break; 1058 } 1059 case BASE_TYPE_LONG: 1060 case BASE_TYPE_ULONG: { 1061 auto hash = FindHashFunction64(hash_name->constant.c_str()); 1062 uint64_t hashed_value = hash(attribute_.c_str()); 1063 e.constant = NumToString(hashed_value); 1064 break; 1065 } 1066 default: 1067 assert(0); 1068 } 1069 NEXT(); 1070 return NoError(); 1071} 1072 1073CheckedError Parser::ParseSingleValue(Value &e) { 1074 // First see if this could be a conversion function: 1075 if (token_ == kTokenIdentifier && *cursor_ == '(') { 1076 auto functionname = attribute_; 1077 NEXT(); 1078 EXPECT('('); 1079 ECHECK(ParseSingleValue(e)); 1080 EXPECT(')'); 1081 #define FLATBUFFERS_FN_DOUBLE(name, op) \ 1082 if (functionname == name) { \ 1083 auto x = strtod(e.constant.c_str(), nullptr); \ 1084 e.constant = NumToString(op); \ 1085 } 1086 FLATBUFFERS_FN_DOUBLE("deg", x / M_PI * 180); 1087 FLATBUFFERS_FN_DOUBLE("rad", x * M_PI / 180); 1088 FLATBUFFERS_FN_DOUBLE("sin", sin(x)); 1089 FLATBUFFERS_FN_DOUBLE("cos", cos(x)); 1090 FLATBUFFERS_FN_DOUBLE("tan", tan(x)); 1091 FLATBUFFERS_FN_DOUBLE("asin", asin(x)); 1092 FLATBUFFERS_FN_DOUBLE("acos", acos(x)); 1093 FLATBUFFERS_FN_DOUBLE("atan", atan(x)); 1094 // TODO(wvo): add more useful conversion functions here. 1095 #undef FLATBUFFERS_FN_DOUBLE 1096 // Then check if this could be a string/identifier enum value: 1097 } else if (e.type.base_type != BASE_TYPE_STRING && 1098 e.type.base_type != BASE_TYPE_NONE && 1099 (token_ == kTokenIdentifier || token_ == kTokenStringConstant)) { 1100 if (IsIdentifierStart(attribute_[0])) { // Enum value. 1101 int64_t val; 1102 ECHECK(ParseEnumFromString(e.type, &val)); 1103 e.constant = NumToString(val); 1104 NEXT(); 1105 } else { // Numeric constant in string. 1106 if (IsInteger(e.type.base_type)) { 1107 char *end; 1108 e.constant = NumToString(StringToInt(attribute_.c_str(), &end)); 1109 if (*end) 1110 return Error("invalid integer: " + attribute_); 1111 } else if (IsFloat(e.type.base_type)) { 1112 char *end; 1113 e.constant = NumToString(strtod(attribute_.c_str(), &end)); 1114 if (*end) 1115 return Error("invalid float: " + attribute_); 1116 } else { 1117 assert(0); // Shouldn't happen, we covered all types. 1118 e.constant = "0"; 1119 } 1120 NEXT(); 1121 } 1122 } else { 1123 bool match = false; 1124 ECHECK(TryTypedValue(kTokenIntegerConstant, 1125 IsScalar(e.type.base_type), 1126 e, 1127 BASE_TYPE_INT, 1128 &match)); 1129 ECHECK(TryTypedValue(kTokenFloatConstant, 1130 IsFloat(e.type.base_type), 1131 e, 1132 BASE_TYPE_FLOAT, 1133 &match)); 1134 ECHECK(TryTypedValue(kTokenStringConstant, 1135 e.type.base_type == BASE_TYPE_STRING, 1136 e, 1137 BASE_TYPE_STRING, 1138 &match)); 1139 if (!match) 1140 return Error("cannot parse value starting with: " + 1141 TokenToStringId(token_)); 1142 } 1143 return NoError(); 1144} 1145 1146StructDef *Parser::LookupCreateStruct(const std::string &name, 1147 bool create_if_new, bool definition) { 1148 std::string qualified_name = namespaces_.back()->GetFullyQualifiedName(name); 1149 // See if it exists pre-declared by an unqualified use. 1150 auto struct_def = structs_.Lookup(name); 1151 if (struct_def && struct_def->predecl) { 1152 if (definition) { 1153 // Make sure it has the current namespace, and is registered under its 1154 // qualified name. 1155 struct_def->defined_namespace = namespaces_.back(); 1156 structs_.Move(name, qualified_name); 1157 } 1158 return struct_def; 1159 } 1160 // See if it exists pre-declared by an qualified use. 1161 struct_def = structs_.Lookup(qualified_name); 1162 if (struct_def && struct_def->predecl) { 1163 if (definition) { 1164 // Make sure it has the current namespace. 1165 struct_def->defined_namespace = namespaces_.back(); 1166 } 1167 return struct_def; 1168 } 1169 if (!definition) { 1170 // Search thru parent namespaces. 1171 for (size_t components = namespaces_.back()->components.size(); 1172 components && !struct_def; components--) { 1173 struct_def = structs_.Lookup( 1174 namespaces_.back()->GetFullyQualifiedName(name, components - 1)); 1175 } 1176 } 1177 if (!struct_def && create_if_new) { 1178 struct_def = new StructDef(); 1179 if (definition) { 1180 structs_.Add(qualified_name, struct_def); 1181 struct_def->name = name; 1182 struct_def->defined_namespace = namespaces_.back(); 1183 } else { 1184 // Not a definition. 1185 // Rather than failing, we create a "pre declared" StructDef, due to 1186 // circular references, and check for errors at the end of parsing. 1187 // It is defined in the root namespace, since we don't know what the 1188 // final namespace will be. 1189 // TODO: maybe safer to use special namespace? 1190 structs_.Add(name, struct_def); 1191 struct_def->name = name; 1192 struct_def->defined_namespace = new Namespace(); 1193 namespaces_.insert(namespaces_.begin(), struct_def->defined_namespace); 1194 } 1195 } 1196 return struct_def; 1197} 1198 1199CheckedError Parser::ParseEnum(bool is_union, EnumDef **dest) { 1200 std::vector<std::string> enum_comment = doc_comment_; 1201 NEXT(); 1202 std::string enum_name = attribute_; 1203 EXPECT(kTokenIdentifier); 1204 auto &enum_def = *new EnumDef(); 1205 enum_def.name = enum_name; 1206 enum_def.file = file_being_parsed_; 1207 enum_def.doc_comment = enum_comment; 1208 enum_def.is_union = is_union; 1209 enum_def.defined_namespace = namespaces_.back(); 1210 if (enums_.Add(namespaces_.back()->GetFullyQualifiedName(enum_name), 1211 &enum_def)) 1212 return Error("enum already exists: " + enum_name); 1213 if (is_union) { 1214 enum_def.underlying_type.base_type = BASE_TYPE_UTYPE; 1215 enum_def.underlying_type.enum_def = &enum_def; 1216 } else { 1217 if (opts.proto_mode) { 1218 enum_def.underlying_type.base_type = BASE_TYPE_INT; 1219 } else { 1220 // Give specialized error message, since this type spec used to 1221 // be optional in the first FlatBuffers release. 1222 if (!Is(':')) { 1223 return Error("must specify the underlying integer type for this" 1224 " enum (e.g. \': short\', which was the default)."); 1225 } else { 1226 NEXT(); 1227 } 1228 // Specify the integer type underlying this enum. 1229 ECHECK(ParseType(enum_def.underlying_type)); 1230 if (!IsInteger(enum_def.underlying_type.base_type)) 1231 return Error("underlying enum type must be integral"); 1232 } 1233 // Make this type refer back to the enum it was derived from. 1234 enum_def.underlying_type.enum_def = &enum_def; 1235 } 1236 ECHECK(ParseMetaData(&enum_def.attributes)); 1237 EXPECT('{'); 1238 if (is_union) enum_def.vals.Add("NONE", new EnumVal("NONE", 0)); 1239 for (;;) { 1240 if (opts.proto_mode && attribute_ == "option") { 1241 ECHECK(ParseProtoOption()); 1242 } else { 1243 auto value_name = attribute_; 1244 auto full_name = value_name; 1245 std::vector<std::string> value_comment = doc_comment_; 1246 EXPECT(kTokenIdentifier); 1247 if (is_union) { 1248 ECHECK(ParseNamespacing(&full_name, &value_name)); 1249 if (opts.union_value_namespacing) { 1250 // Since we can't namespace the actual enum identifiers, turn 1251 // namespace parts into part of the identifier. 1252 value_name = full_name; 1253 std::replace(value_name.begin(), value_name.end(), '.', '_'); 1254 } 1255 } 1256 auto prevsize = enum_def.vals.vec.size(); 1257 auto value = enum_def.vals.vec.size() 1258 ? enum_def.vals.vec.back()->value + 1 1259 : 0; 1260 auto &ev = *new EnumVal(value_name, value); 1261 if (enum_def.vals.Add(value_name, &ev)) 1262 return Error("enum value already exists: " + value_name); 1263 ev.doc_comment = value_comment; 1264 if (is_union) { 1265 ev.struct_def = LookupCreateStruct(full_name); 1266 } 1267 if (Is('=')) { 1268 NEXT(); 1269 ev.value = StringToInt(attribute_.c_str()); 1270 EXPECT(kTokenIntegerConstant); 1271 if (!opts.proto_mode && prevsize && 1272 enum_def.vals.vec[prevsize - 1]->value >= ev.value) 1273 return Error("enum values must be specified in ascending order"); 1274 } 1275 if (opts.proto_mode && Is('[')) { 1276 NEXT(); 1277 // ignore attributes on enums. 1278 while (token_ != ']') NEXT(); 1279 NEXT(); 1280 } 1281 } 1282 if (!Is(opts.proto_mode ? ';' : ',')) break; 1283 NEXT(); 1284 if (Is('}')) break; 1285 } 1286 EXPECT('}'); 1287 if (enum_def.attributes.Lookup("bit_flags")) { 1288 for (auto it = enum_def.vals.vec.begin(); it != enum_def.vals.vec.end(); 1289 ++it) { 1290 if (static_cast<size_t>((*it)->value) >= 1291 SizeOf(enum_def.underlying_type.base_type) * 8) 1292 return Error("bit flag out of range of underlying integral type"); 1293 (*it)->value = 1LL << (*it)->value; 1294 } 1295 } 1296 if (dest) *dest = &enum_def; 1297 return NoError(); 1298} 1299 1300CheckedError Parser::StartStruct(const std::string &name, StructDef **dest) { 1301 auto &struct_def = *LookupCreateStruct(name, true, true); 1302 if (!struct_def.predecl) return Error("datatype already exists: " + name); 1303 struct_def.predecl = false; 1304 struct_def.name = name; 1305 struct_def.file = file_being_parsed_; 1306 // Move this struct to the back of the vector just in case it was predeclared, 1307 // to preserve declaration order. 1308 *remove(structs_.vec.begin(), structs_.vec.end(), &struct_def) = &struct_def; 1309 *dest = &struct_def; 1310 return NoError(); 1311} 1312 1313CheckedError Parser::CheckClash(std::vector<FieldDef*> &fields, 1314 StructDef *struct_def, 1315 const char *suffix, 1316 BaseType basetype) { 1317 auto len = strlen(suffix); 1318 for (auto it = fields.begin(); it != fields.end(); ++it) { 1319 auto &fname = (*it)->name; 1320 if (fname.length() > len && 1321 fname.compare(fname.length() - len, len, suffix) == 0 && 1322 (*it)->value.type.base_type != BASE_TYPE_UTYPE) { 1323 auto field = struct_def->fields.Lookup( 1324 fname.substr(0, fname.length() - len)); 1325 if (field && field->value.type.base_type == basetype) 1326 return Error("Field " + fname + 1327 " would clash with generated functions for field " + 1328 field->name); 1329 } 1330 } 1331 return NoError(); 1332} 1333 1334static bool compareFieldDefs(const FieldDef *a, const FieldDef *b) { 1335 auto a_id = atoi(a->attributes.Lookup("id")->constant.c_str()); 1336 auto b_id = atoi(b->attributes.Lookup("id")->constant.c_str()); 1337 return a_id < b_id; 1338} 1339 1340CheckedError Parser::ParseDecl() { 1341 std::vector<std::string> dc = doc_comment_; 1342 bool fixed = Is(kTokenStruct); 1343 if (fixed) NEXT() else EXPECT(kTokenTable); 1344 std::string name = attribute_; 1345 EXPECT(kTokenIdentifier); 1346 StructDef *struct_def; 1347 ECHECK(StartStruct(name, &struct_def)); 1348 struct_def->doc_comment = dc; 1349 struct_def->fixed = fixed; 1350 ECHECK(ParseMetaData(&struct_def->attributes)); 1351 struct_def->sortbysize = 1352 struct_def->attributes.Lookup("original_order") == nullptr && !fixed; 1353 EXPECT('{'); 1354 while (token_ != '}') ECHECK(ParseField(*struct_def)); 1355 auto force_align = struct_def->attributes.Lookup("force_align"); 1356 if (fixed && force_align) { 1357 auto align = static_cast<size_t>(atoi(force_align->constant.c_str())); 1358 if (force_align->type.base_type != BASE_TYPE_INT || 1359 align < struct_def->minalign || 1360 align > FLATBUFFERS_MAX_ALIGNMENT || 1361 align & (align - 1)) 1362 return Error("force_align must be a power of two integer ranging from the" 1363 "struct\'s natural alignment to " + 1364 NumToString(FLATBUFFERS_MAX_ALIGNMENT)); 1365 struct_def->minalign = align; 1366 } 1367 struct_def->PadLastField(struct_def->minalign); 1368 // Check if this is a table that has manual id assignments 1369 auto &fields = struct_def->fields.vec; 1370 if (!struct_def->fixed && fields.size()) { 1371 size_t num_id_fields = 0; 1372 for (auto it = fields.begin(); it != fields.end(); ++it) { 1373 if ((*it)->attributes.Lookup("id")) num_id_fields++; 1374 } 1375 // If any fields have ids.. 1376 if (num_id_fields) { 1377 // Then all fields must have them. 1378 if (num_id_fields != fields.size()) 1379 return Error( 1380 "either all fields or no fields must have an 'id' attribute"); 1381 // Simply sort by id, then the fields are the same as if no ids had 1382 // been specified. 1383 std::sort(fields.begin(), fields.end(), compareFieldDefs); 1384 // Verify we have a contiguous set, and reassign vtable offsets. 1385 for (int i = 0; i < static_cast<int>(fields.size()); i++) { 1386 if (i != atoi(fields[i]->attributes.Lookup("id")->constant.c_str())) 1387 return Error("field id\'s must be consecutive from 0, id " + 1388 NumToString(i) + " missing or set twice"); 1389 fields[i]->value.offset = FieldIndexToOffset(static_cast<voffset_t>(i)); 1390 } 1391 } 1392 } 1393 1394 ECHECK(CheckClash(fields, struct_def, UnionTypeFieldSuffix(), 1395 BASE_TYPE_UNION)); 1396 ECHECK(CheckClash(fields, struct_def, "Type", BASE_TYPE_UNION)); 1397 ECHECK(CheckClash(fields, struct_def, "_length", BASE_TYPE_VECTOR)); 1398 ECHECK(CheckClash(fields, struct_def, "Length", BASE_TYPE_VECTOR)); 1399 ECHECK(CheckClash(fields, struct_def, "_byte_vector", BASE_TYPE_STRING)); 1400 ECHECK(CheckClash(fields, struct_def, "ByteVector", BASE_TYPE_STRING)); 1401 EXPECT('}'); 1402 return NoError(); 1403} 1404 1405CheckedError Parser::ParseService() { 1406 std::vector<std::string> service_comment = doc_comment_; 1407 NEXT(); 1408 auto service_name = attribute_; 1409 EXPECT(kTokenIdentifier); 1410 auto &service_def = *new ServiceDef(); 1411 service_def.name = service_name; 1412 service_def.file = file_being_parsed_; 1413 service_def.doc_comment = service_comment; 1414 service_def.defined_namespace = namespaces_.back(); 1415 if (services_.Add(namespaces_.back()->GetFullyQualifiedName(service_name), 1416 &service_def)) 1417 return Error("service already exists: " + service_name); 1418 ECHECK(ParseMetaData(&service_def.attributes)); 1419 EXPECT('{'); 1420 do { 1421 auto rpc_name = attribute_; 1422 EXPECT(kTokenIdentifier); 1423 EXPECT('('); 1424 Type reqtype, resptype; 1425 ECHECK(ParseTypeIdent(reqtype)); 1426 EXPECT(')'); 1427 EXPECT(':'); 1428 ECHECK(ParseTypeIdent(resptype)); 1429 if (reqtype.base_type != BASE_TYPE_STRUCT || reqtype.struct_def->fixed || 1430 resptype.base_type != BASE_TYPE_STRUCT || resptype.struct_def->fixed) 1431 return Error("rpc request and response types must be tables"); 1432 auto &rpc = *new RPCCall(); 1433 rpc.name = rpc_name; 1434 rpc.request = reqtype.struct_def; 1435 rpc.response = resptype.struct_def; 1436 if (service_def.calls.Add(rpc_name, &rpc)) 1437 return Error("rpc already exists: " + rpc_name); 1438 ECHECK(ParseMetaData(&rpc.attributes)); 1439 EXPECT(';'); 1440 } while (token_ != '}'); 1441 NEXT(); 1442 return NoError(); 1443} 1444 1445bool Parser::SetRootType(const char *name) { 1446 root_struct_def_ = structs_.Lookup(name); 1447 if (!root_struct_def_) 1448 root_struct_def_ = structs_.Lookup( 1449 namespaces_.back()->GetFullyQualifiedName(name)); 1450 return root_struct_def_ != nullptr; 1451} 1452 1453void Parser::MarkGenerated() { 1454 // This function marks all existing definitions as having already 1455 // been generated, which signals no code for included files should be 1456 // generated. 1457 for (auto it = enums_.vec.begin(); 1458 it != enums_.vec.end(); ++it) { 1459 (*it)->generated = true; 1460 } 1461 for (auto it = structs_.vec.begin(); 1462 it != structs_.vec.end(); ++it) { 1463 (*it)->generated = true; 1464 } 1465 for (auto it = services_.vec.begin(); 1466 it != services_.vec.end(); ++it) { 1467 (*it)->generated = true; 1468 } 1469} 1470 1471CheckedError Parser::ParseNamespace() { 1472 NEXT(); 1473 auto ns = new Namespace(); 1474 namespaces_.push_back(ns); 1475 if (token_ != ';') { 1476 for (;;) { 1477 ns->components.push_back(attribute_); 1478 EXPECT(kTokenIdentifier); 1479 if (Is('.')) NEXT() else break; 1480 } 1481 } 1482 EXPECT(';'); 1483 return NoError(); 1484} 1485 1486static bool compareEnumVals(const EnumVal *a, const EnumVal* b) { 1487 return a->value < b->value; 1488} 1489 1490// Best effort parsing of .proto declarations, with the aim to turn them 1491// in the closest corresponding FlatBuffer equivalent. 1492// We parse everything as identifiers instead of keywords, since we don't 1493// want protobuf keywords to become invalid identifiers in FlatBuffers. 1494CheckedError Parser::ParseProtoDecl() { 1495 bool isextend = attribute_ == "extend"; 1496 if (attribute_ == "package") { 1497 // These are identical in syntax to FlatBuffer's namespace decl. 1498 ECHECK(ParseNamespace()); 1499 } else if (attribute_ == "message" || isextend) { 1500 std::vector<std::string> struct_comment = doc_comment_; 1501 NEXT(); 1502 StructDef *struct_def = nullptr; 1503 if (isextend) { 1504 if (Is('.')) NEXT(); // qualified names may start with a . ? 1505 auto id = attribute_; 1506 EXPECT(kTokenIdentifier); 1507 ECHECK(ParseNamespacing(&id, nullptr)); 1508 struct_def = LookupCreateStruct(id, false); 1509 if (!struct_def) 1510 return Error("cannot extend unknown message type: " + id); 1511 } else { 1512 std::string name = attribute_; 1513 EXPECT(kTokenIdentifier); 1514 ECHECK(StartStruct(name, &struct_def)); 1515 // Since message definitions can be nested, we create a new namespace. 1516 auto ns = new Namespace(); 1517 // Copy of current namespace. 1518 *ns = *namespaces_.back(); 1519 // But with current message name. 1520 ns->components.push_back(name); 1521 namespaces_.push_back(ns); 1522 } 1523 struct_def->doc_comment = struct_comment; 1524 ECHECK(ParseProtoFields(struct_def, isextend, false)); 1525 if (!isextend) { 1526 // We have to remove the nested namespace, but we can't just throw it 1527 // away, so put it at the beginning of the vector. 1528 auto ns = namespaces_.back(); 1529 namespaces_.pop_back(); 1530 namespaces_.insert(namespaces_.begin(), ns); 1531 } 1532 if (Is(';')) NEXT(); 1533 } else if (attribute_ == "enum") { 1534 // These are almost the same, just with different terminator: 1535 EnumDef *enum_def; 1536 ECHECK(ParseEnum(false, &enum_def)); 1537 if (Is(';')) NEXT(); 1538 // Protobuf allows them to be specified in any order, so sort afterwards. 1539 auto &v = enum_def->vals.vec; 1540 std::sort(v.begin(), v.end(), compareEnumVals); 1541 1542 // Temp: remove any duplicates, as .fbs files can't handle them. 1543 for (auto it = v.begin(); it != v.end(); ) { 1544 if (it != v.begin() && it[0]->value == it[-1]->value) it = v.erase(it); 1545 else ++it; 1546 } 1547 } else if (attribute_ == "syntax") { // Skip these. 1548 NEXT(); 1549 EXPECT('='); 1550 EXPECT(kTokenStringConstant); 1551 EXPECT(';'); 1552 } else if (attribute_ == "option") { // Skip these. 1553 ECHECK(ParseProtoOption()); 1554 EXPECT(';'); 1555 } else if (attribute_ == "service") { // Skip these. 1556 NEXT(); 1557 EXPECT(kTokenIdentifier); 1558 ECHECK(ParseProtoCurliesOrIdent()); 1559 } else { 1560 return Error("don\'t know how to parse .proto declaration starting with " + 1561 TokenToStringId(token_)); 1562 } 1563 return NoError(); 1564} 1565 1566CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend, 1567 bool inside_oneof) { 1568 EXPECT('{'); 1569 while (token_ != '}') { 1570 if (attribute_ == "message" || attribute_ == "extend" || 1571 attribute_ == "enum") { 1572 // Nested declarations. 1573 ECHECK(ParseProtoDecl()); 1574 } else if (attribute_ == "extensions") { // Skip these. 1575 NEXT(); 1576 EXPECT(kTokenIntegerConstant); 1577 if (Is(kTokenIdentifier)) { 1578 NEXT(); // to 1579 NEXT(); // num 1580 } 1581 EXPECT(';'); 1582 } else if (attribute_ == "option") { // Skip these. 1583 ECHECK(ParseProtoOption()); 1584 EXPECT(';'); 1585 } else if (attribute_ == "reserved") { // Skip these. 1586 NEXT(); 1587 EXPECT(kTokenIntegerConstant); 1588 while (Is(',')) { NEXT(); EXPECT(kTokenIntegerConstant); } 1589 EXPECT(';'); 1590 } else { 1591 std::vector<std::string> field_comment = doc_comment_; 1592 // Parse the qualifier. 1593 bool required = false; 1594 bool repeated = false; 1595 bool oneof = false; 1596 if (!inside_oneof) { 1597 if (attribute_ == "optional") { 1598 // This is the default. 1599 EXPECT(kTokenIdentifier); 1600 } else if (attribute_ == "required") { 1601 required = true; 1602 EXPECT(kTokenIdentifier); 1603 } else if (attribute_ == "repeated") { 1604 repeated = true; 1605 EXPECT(kTokenIdentifier); 1606 } else if (attribute_ == "oneof") { 1607 oneof = true; 1608 EXPECT(kTokenIdentifier); 1609 } else { 1610 // can't error, proto3 allows decls without any of the above. 1611 } 1612 } 1613 StructDef *anonymous_struct = nullptr; 1614 Type type; 1615 if (attribute_ == "group" || oneof) { 1616 if (!oneof) EXPECT(kTokenIdentifier); 1617 auto name = "Anonymous" + NumToString(anonymous_counter++); 1618 ECHECK(StartStruct(name, &anonymous_struct)); 1619 type = Type(BASE_TYPE_STRUCT, anonymous_struct); 1620 } else { 1621 ECHECK(ParseTypeFromProtoType(&type)); 1622 } 1623 // Repeated elements get mapped to a vector. 1624 if (repeated) { 1625 type.element = type.base_type; 1626 type.base_type = BASE_TYPE_VECTOR; 1627 } 1628 std::string name = attribute_; 1629 // Protos may use our keywords "attribute" & "namespace" as an identifier. 1630 if (Is(kTokenAttribute) || Is(kTokenNameSpace)) { 1631 NEXT(); 1632 // TODO: simpler to just not make these keywords? 1633 name += "_"; // Have to make it not a keyword. 1634 } else { 1635 EXPECT(kTokenIdentifier); 1636 } 1637 if (!oneof) { 1638 // Parse the field id. Since we're just translating schemas, not 1639 // any kind of binary compatibility, we can safely ignore these, and 1640 // assign our own. 1641 EXPECT('='); 1642 EXPECT(kTokenIntegerConstant); 1643 } 1644 FieldDef *field = nullptr; 1645 if (isextend) { 1646 // We allow a field to be re-defined when extending. 1647 // TODO: are there situations where that is problematic? 1648 field = struct_def->fields.Lookup(name); 1649 } 1650 if (!field) ECHECK(AddField(*struct_def, name, type, &field)); 1651 field->doc_comment = field_comment; 1652 if (!IsScalar(type.base_type)) field->required = required; 1653 // See if there's a default specified. 1654 if (Is('[')) { 1655 NEXT(); 1656 for (;;) { 1657 auto key = attribute_; 1658 ECHECK(ParseProtoKey()); 1659 EXPECT('='); 1660 auto val = attribute_; 1661 ECHECK(ParseProtoCurliesOrIdent()); 1662 if (key == "default") { 1663 // Temp: skip non-numeric defaults (enums). 1664 auto numeric = strpbrk(val.c_str(), "0123456789-+."); 1665 if (IsScalar(type.base_type) && numeric == val.c_str()) 1666 field->value.constant = val; 1667 } else if (key == "deprecated") { 1668 field->deprecated = val == "true"; 1669 } 1670 if (!Is(',')) break; 1671 NEXT(); 1672 } 1673 EXPECT(']'); 1674 } 1675 if (anonymous_struct) { 1676 ECHECK(ParseProtoFields(anonymous_struct, false, oneof)); 1677 if (Is(';')) NEXT(); 1678 } else { 1679 EXPECT(';'); 1680 } 1681 } 1682 } 1683 NEXT(); 1684 return NoError(); 1685} 1686 1687CheckedError Parser::ParseProtoKey() { 1688 if (token_ == '(') { 1689 NEXT(); 1690 // Skip "(a.b)" style custom attributes. 1691 while (token_ == '.' || token_ == kTokenIdentifier) NEXT(); 1692 EXPECT(')'); 1693 while (Is('.')) { NEXT(); EXPECT(kTokenIdentifier); } 1694 } else { 1695 EXPECT(kTokenIdentifier); 1696 } 1697 return NoError(); 1698} 1699 1700CheckedError Parser::ParseProtoCurliesOrIdent() { 1701 if (Is('{')) { 1702 NEXT(); 1703 for (int nesting = 1; nesting; ) { 1704 if (token_ == '{') nesting++; 1705 else if (token_ == '}') nesting--; 1706 NEXT(); 1707 } 1708 } else { 1709 NEXT(); // Any single token. 1710 } 1711 return NoError(); 1712} 1713 1714CheckedError Parser::ParseProtoOption() { 1715 NEXT(); 1716 ECHECK(ParseProtoKey()); 1717 EXPECT('='); 1718 ECHECK(ParseProtoCurliesOrIdent()); 1719 return NoError(); 1720} 1721 1722// Parse a protobuf type, and map it to the corresponding FlatBuffer one. 1723CheckedError Parser::ParseTypeFromProtoType(Type *type) { 1724 struct type_lookup { const char *proto_type; BaseType fb_type; }; 1725 static type_lookup lookup[] = { 1726 { "float", BASE_TYPE_FLOAT }, { "double", BASE_TYPE_DOUBLE }, 1727 { "int32", BASE_TYPE_INT }, { "int64", BASE_TYPE_LONG }, 1728 { "uint32", BASE_TYPE_UINT }, { "uint64", BASE_TYPE_ULONG }, 1729 { "sint32", BASE_TYPE_INT }, { "sint64", BASE_TYPE_LONG }, 1730 { "fixed32", BASE_TYPE_UINT }, { "fixed64", BASE_TYPE_ULONG }, 1731 { "sfixed32", BASE_TYPE_INT }, { "sfixed64", BASE_TYPE_LONG }, 1732 { "bool", BASE_TYPE_BOOL }, 1733 { "string", BASE_TYPE_STRING }, 1734 { "bytes", BASE_TYPE_STRING }, 1735 { nullptr, BASE_TYPE_NONE } 1736 }; 1737 for (auto tl = lookup; tl->proto_type; tl++) { 1738 if (attribute_ == tl->proto_type) { 1739 type->base_type = tl->fb_type; 1740 NEXT(); 1741 return NoError(); 1742 } 1743 } 1744 if (Is('.')) NEXT(); // qualified names may start with a . ? 1745 ECHECK(ParseTypeIdent(*type)); 1746 return NoError(); 1747} 1748 1749CheckedError Parser::SkipAnyJsonValue() { 1750 switch (token_) { 1751 case '{': 1752 ECHECK(SkipJsonObject()); 1753 break; 1754 case kTokenStringConstant: 1755 ECHECK(SkipJsonString()); 1756 break; 1757 case '[': 1758 ECHECK(SkipJsonArray()); 1759 break; 1760 case kTokenIntegerConstant: 1761 EXPECT(kTokenIntegerConstant); 1762 break; 1763 case kTokenFloatConstant: 1764 EXPECT(kTokenFloatConstant); 1765 break; 1766 default: 1767 return Error(std::string("Unexpected token:") + std::string(1, static_cast<char>(token_))); 1768 } 1769 return NoError(); 1770} 1771 1772CheckedError Parser::SkipJsonObject() { 1773 EXPECT('{'); 1774 size_t fieldn = 0; 1775 1776 for (;;) { 1777 if ((!opts.strict_json || !fieldn) && Is('}')) break; 1778 1779 if (!Is(kTokenStringConstant)) { 1780 EXPECT(opts.strict_json ? kTokenStringConstant : kTokenIdentifier); 1781 } 1782 else { 1783 NEXT(); 1784 } 1785 1786 EXPECT(':'); 1787 ECHECK(SkipAnyJsonValue()); 1788 fieldn++; 1789 1790 if (Is('}')) break; 1791 EXPECT(','); 1792 } 1793 1794 NEXT(); 1795 return NoError(); 1796} 1797 1798CheckedError Parser::SkipJsonArray() { 1799 EXPECT('['); 1800 1801 for (;;) { 1802 if (Is(']')) break; 1803 1804 ECHECK(SkipAnyJsonValue()); 1805 1806 if (Is(']')) break; 1807 EXPECT(','); 1808 } 1809 1810 NEXT(); 1811 return NoError(); 1812} 1813 1814CheckedError Parser::SkipJsonString() { 1815 EXPECT(kTokenStringConstant); 1816 return NoError(); 1817} 1818 1819bool Parser::Parse(const char *source, const char **include_paths, 1820 const char *source_filename) { 1821 return !DoParse(source, include_paths, source_filename).Check(); 1822} 1823 1824CheckedError Parser::DoParse(const char *source, const char **include_paths, 1825 const char *source_filename) { 1826 file_being_parsed_ = source_filename ? source_filename : ""; 1827 if (source_filename && 1828 included_files_.find(source_filename) == included_files_.end()) { 1829 included_files_[source_filename] = true; 1830 files_included_per_file_[source_filename] = std::set<std::string>(); 1831 } 1832 if (!include_paths) { 1833 static const char *current_directory[] = { "", nullptr }; 1834 include_paths = current_directory; 1835 } 1836 source_ = cursor_ = source; 1837 line_ = 1; 1838 error_.clear(); 1839 builder_.Clear(); 1840 // Start with a blank namespace just in case this file doesn't have one. 1841 namespaces_.push_back(new Namespace()); 1842 ECHECK(SkipByteOrderMark()); 1843 NEXT(); 1844 // Includes must come before type declarations: 1845 for (;;) { 1846 // Parse pre-include proto statements if any: 1847 if (opts.proto_mode && 1848 (attribute_ == "option" || attribute_ == "syntax" || 1849 attribute_ == "package")) { 1850 ECHECK(ParseProtoDecl()); 1851 } else if (Is(kTokenInclude) || 1852 (opts.proto_mode && 1853 attribute_ == "import" && 1854 Is(kTokenIdentifier))) { 1855 NEXT(); 1856 if (opts.proto_mode && attribute_ == "public") NEXT(); 1857 auto name = attribute_; 1858 EXPECT(kTokenStringConstant); 1859 // Look for the file in include_paths. 1860 std::string filepath; 1861 for (auto paths = include_paths; paths && *paths; paths++) { 1862 filepath = flatbuffers::ConCatPathFileName(*paths, name); 1863 if(FileExists(filepath.c_str())) break; 1864 } 1865 if (filepath.empty()) 1866 return Error("unable to locate include file: " + name); 1867 if (source_filename) 1868 files_included_per_file_[source_filename].insert(filepath); 1869 if (included_files_.find(filepath) == included_files_.end()) { 1870 // We found an include file that we have not parsed yet. 1871 // Load it and parse it. 1872 std::string contents; 1873 if (!LoadFile(filepath.c_str(), true, &contents)) 1874 return Error("unable to load include file: " + name); 1875 ECHECK(DoParse(contents.c_str(), include_paths, filepath.c_str())); 1876 // We generally do not want to output code for any included files: 1877 if (!opts.generate_all) MarkGenerated(); 1878 // This is the easiest way to continue this file after an include: 1879 // instead of saving and restoring all the state, we simply start the 1880 // file anew. This will cause it to encounter the same include 1881 // statement again, but this time it will skip it, because it was 1882 // entered into included_files_. 1883 // This is recursive, but only go as deep as the number of include 1884 // statements. 1885 return DoParse(source, include_paths, source_filename); 1886 } 1887 EXPECT(';'); 1888 } else { 1889 break; 1890 } 1891 } 1892 // Now parse all other kinds of declarations: 1893 while (token_ != kTokenEof) { 1894 if (opts.proto_mode) { 1895 ECHECK(ParseProtoDecl()); 1896 } else if (token_ == kTokenNameSpace) { 1897 ECHECK(ParseNamespace()); 1898 } else if (token_ == '{') { 1899 if (!root_struct_def_) 1900 return Error("no root type set to parse json with"); 1901 if (builder_.GetSize()) { 1902 return Error("cannot have more than one json object in a file"); 1903 } 1904 uoffset_t toff; 1905 ECHECK(ParseTable(*root_struct_def_, nullptr, &toff)); 1906 builder_.Finish(Offset<Table>(toff), 1907 file_identifier_.length() ? file_identifier_.c_str() : nullptr); 1908 } else if (token_ == kTokenEnum) { 1909 ECHECK(ParseEnum(false, nullptr)); 1910 } else if (token_ == kTokenUnion) { 1911 ECHECK(ParseEnum(true, nullptr)); 1912 } else if (token_ == kTokenRootType) { 1913 NEXT(); 1914 auto root_type = attribute_; 1915 EXPECT(kTokenIdentifier); 1916 ECHECK(ParseNamespacing(&root_type, nullptr)); 1917 if (!SetRootType(root_type.c_str())) 1918 return Error("unknown root type: " + root_type); 1919 if (root_struct_def_->fixed) 1920 return Error("root type must be a table"); 1921 EXPECT(';'); 1922 } else if (token_ == kTokenFileIdentifier) { 1923 NEXT(); 1924 file_identifier_ = attribute_; 1925 EXPECT(kTokenStringConstant); 1926 if (file_identifier_.length() != 1927 FlatBufferBuilder::kFileIdentifierLength) 1928 return Error("file_identifier must be exactly " + 1929 NumToString(FlatBufferBuilder::kFileIdentifierLength) + 1930 " characters"); 1931 EXPECT(';'); 1932 } else if (token_ == kTokenFileExtension) { 1933 NEXT(); 1934 file_extension_ = attribute_; 1935 EXPECT(kTokenStringConstant); 1936 EXPECT(';'); 1937 } else if(token_ == kTokenInclude) { 1938 return Error("includes must come before declarations"); 1939 } else if(token_ == kTokenAttribute) { 1940 NEXT(); 1941 auto name = attribute_; 1942 EXPECT(kTokenStringConstant); 1943 EXPECT(';'); 1944 known_attributes_[name] = false; 1945 } else if (token_ == kTokenService) { 1946 ECHECK(ParseService()); 1947 } else { 1948 ECHECK(ParseDecl()); 1949 } 1950 } 1951 for (auto it = structs_.vec.begin(); it != structs_.vec.end(); ++it) { 1952 if ((*it)->predecl) { 1953 return Error("type referenced but not defined: " + (*it)->name); 1954 } 1955 } 1956 for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) { 1957 auto &enum_def = **it; 1958 if (enum_def.is_union) { 1959 for (auto val_it = enum_def.vals.vec.begin(); 1960 val_it != enum_def.vals.vec.end(); 1961 ++val_it) { 1962 auto &val = **val_it; 1963 if (val.struct_def && val.struct_def->fixed) 1964 return Error("only tables can be union elements: " + val.name); 1965 } 1966 } 1967 } 1968 return NoError(); 1969} 1970 1971std::set<std::string> Parser::GetIncludedFilesRecursive( 1972 const std::string &file_name) const { 1973 std::set<std::string> included_files; 1974 std::list<std::string> to_process; 1975 1976 if (file_name.empty()) return included_files; 1977 to_process.push_back(file_name); 1978 1979 while (!to_process.empty()) { 1980 std::string current = to_process.front(); 1981 to_process.pop_front(); 1982 included_files.insert(current); 1983 1984 auto new_files = files_included_per_file_.at(current); 1985 for (auto it = new_files.begin(); it != new_files.end(); ++it) { 1986 if (included_files.find(*it) == included_files.end()) 1987 to_process.push_back(*it); 1988 } 1989 } 1990 1991 return included_files; 1992} 1993 1994// Schema serialization functionality: 1995 1996template<typename T> bool compareName(const T* a, const T* b) { 1997 return a->defined_namespace->GetFullyQualifiedName(a->name) 1998 < b->defined_namespace->GetFullyQualifiedName(b->name); 1999} 2000 2001template<typename T> void AssignIndices(const std::vector<T *> &defvec) { 2002 // Pre-sort these vectors, such that we can set the correct indices for them. 2003 auto vec = defvec; 2004 std::sort(vec.begin(), vec.end(), compareName<T>); 2005 for (int i = 0; i < static_cast<int>(vec.size()); i++) vec[i]->index = i; 2006} 2007 2008void Parser::Serialize() { 2009 builder_.Clear(); 2010 AssignIndices(structs_.vec); 2011 AssignIndices(enums_.vec); 2012 std::vector<Offset<reflection::Object>> object_offsets; 2013 for (auto it = structs_.vec.begin(); it != structs_.vec.end(); ++it) { 2014 auto offset = (*it)->Serialize(&builder_, *this); 2015 object_offsets.push_back(offset); 2016 (*it)->serialized_location = offset.o; 2017 } 2018 std::vector<Offset<reflection::Enum>> enum_offsets; 2019 for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) { 2020 auto offset = (*it)->Serialize(&builder_, *this); 2021 enum_offsets.push_back(offset); 2022 (*it)->serialized_location = offset.o; 2023 } 2024 auto schema_offset = reflection::CreateSchema( 2025 builder_, 2026 builder_.CreateVectorOfSortedTables(&object_offsets), 2027 builder_.CreateVectorOfSortedTables(&enum_offsets), 2028 builder_.CreateString(file_identifier_), 2029 builder_.CreateString(file_extension_), 2030 root_struct_def_ 2031 ? root_struct_def_->serialized_location 2032 : 0); 2033 builder_.Finish(schema_offset, reflection::SchemaIdentifier()); 2034} 2035 2036Offset<reflection::Object> StructDef::Serialize(FlatBufferBuilder *builder, 2037 const Parser &parser) const { 2038 std::vector<Offset<reflection::Field>> field_offsets; 2039 for (auto it = fields.vec.begin(); it != fields.vec.end(); ++it) { 2040 field_offsets.push_back( 2041 (*it)->Serialize(builder, 2042 static_cast<uint16_t>(it - fields.vec.begin()), parser)); 2043 } 2044 auto qualified_name = defined_namespace->GetFullyQualifiedName(name); 2045 return reflection::CreateObject(*builder, 2046 builder->CreateString(qualified_name), 2047 builder->CreateVectorOfSortedTables( 2048 &field_offsets), 2049 fixed, 2050 static_cast<int>(minalign), 2051 static_cast<int>(bytesize), 2052 SerializeAttributes(builder, parser)); 2053} 2054 2055Offset<reflection::Field> FieldDef::Serialize(FlatBufferBuilder *builder, 2056 uint16_t id, 2057 const Parser &parser) const { 2058 return reflection::CreateField(*builder, 2059 builder->CreateString(name), 2060 value.type.Serialize(builder), 2061 id, 2062 value.offset, 2063 IsInteger(value.type.base_type) 2064 ? StringToInt(value.constant.c_str()) 2065 : 0, 2066 IsFloat(value.type.base_type) 2067 ? strtod(value.constant.c_str(), nullptr) 2068 : 0.0, 2069 deprecated, 2070 required, 2071 key, 2072 SerializeAttributes(builder, parser)); 2073 // TODO: value.constant is almost always "0", we could save quite a bit of 2074 // space by sharing it. Same for common values of value.type. 2075} 2076 2077Offset<reflection::Enum> EnumDef::Serialize(FlatBufferBuilder *builder, 2078 const Parser &parser) const { 2079 std::vector<Offset<reflection::EnumVal>> enumval_offsets; 2080 for (auto it = vals.vec.begin(); it != vals.vec.end(); ++it) { 2081 enumval_offsets.push_back((*it)->Serialize(builder)); 2082 } 2083 auto qualified_name = defined_namespace->GetFullyQualifiedName(name); 2084 return reflection::CreateEnum(*builder, 2085 builder->CreateString(qualified_name), 2086 builder->CreateVector(enumval_offsets), 2087 is_union, 2088 underlying_type.Serialize(builder), 2089 SerializeAttributes(builder, parser)); 2090} 2091 2092Offset<reflection::EnumVal> EnumVal::Serialize(FlatBufferBuilder *builder) const 2093 { 2094 return reflection::CreateEnumVal(*builder, 2095 builder->CreateString(name), 2096 value, 2097 struct_def 2098 ? struct_def->serialized_location 2099 : 0); 2100} 2101 2102Offset<reflection::Type> Type::Serialize(FlatBufferBuilder *builder) const { 2103 return reflection::CreateType(*builder, 2104 static_cast<reflection::BaseType>(base_type), 2105 static_cast<reflection::BaseType>(element), 2106 struct_def ? struct_def->index : 2107 (enum_def ? enum_def->index : -1)); 2108} 2109 2110flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset< 2111 reflection::KeyValue>>> 2112 Definition::SerializeAttributes(FlatBufferBuilder *builder, 2113 const Parser &parser) const { 2114 std::vector<flatbuffers::Offset<reflection::KeyValue>> attrs; 2115 for (auto kv = attributes.dict.begin(); kv != attributes.dict.end(); ++kv) { 2116 auto it = parser.known_attributes_.find(kv->first); 2117 assert(it != parser.known_attributes_.end()); 2118 if (!it->second) { // Custom attribute. 2119 attrs.push_back( 2120 reflection::CreateKeyValue(*builder, builder->CreateString(kv->first), 2121 builder->CreateString( 2122 kv->second->constant))); 2123 } 2124 } 2125 if (attrs.size()) { 2126 return builder->CreateVectorOfSortedTables(&attrs); 2127 } else { 2128 return 0; 2129 } 2130} 2131 2132std::string Parser::ConformTo(const Parser &base) { 2133 for (auto sit = structs_.vec.begin(); sit != structs_.vec.end(); ++sit) { 2134 auto &struct_def = **sit; 2135 auto qualified_name = 2136 struct_def.defined_namespace->GetFullyQualifiedName(struct_def.name); 2137 auto struct_def_base = base.structs_.Lookup(qualified_name); 2138 if (!struct_def_base) continue; 2139 for (auto fit = struct_def.fields.vec.begin(); 2140 fit != struct_def.fields.vec.end(); ++fit) { 2141 auto &field = **fit; 2142 auto field_base = struct_def_base->fields.Lookup(field.name); 2143 if (field_base) { 2144 if (field.value.offset != field_base->value.offset) 2145 return "offsets differ for field: " + field.name; 2146 if (field.value.constant != field_base->value.constant) 2147 return "defaults differ for field: " + field.name; 2148 if (!EqualByName(field.value.type, field_base->value.type)) 2149 return "types differ for field: " + field.name; 2150 } else { 2151 // Doesn't have to exist, deleting fields is fine. 2152 // But we should check if there is a field that has the same offset 2153 // but is incompatible (in the case of field renaming). 2154 for (auto fbit = struct_def_base->fields.vec.begin(); 2155 fbit != struct_def_base->fields.vec.end(); ++fbit) { 2156 field_base = *fbit; 2157 if (field.value.offset == field_base->value.offset) { 2158 if (!EqualByName(field.value.type, field_base->value.type)) 2159 return "field renamed to different type: " + field.name; 2160 break; 2161 } 2162 } 2163 } 2164 } 2165 } 2166 for (auto eit = enums_.vec.begin(); eit != enums_.vec.end(); ++eit) { 2167 auto &enum_def = **eit; 2168 auto qualified_name = 2169 enum_def.defined_namespace->GetFullyQualifiedName(enum_def.name); 2170 auto enum_def_base = base.enums_.Lookup(qualified_name); 2171 if (!enum_def_base) continue; 2172 for (auto evit = enum_def.vals.vec.begin(); 2173 evit != enum_def.vals.vec.end(); ++evit) { 2174 auto &enum_val = **evit; 2175 auto enum_val_base = enum_def_base->vals.Lookup(enum_val.name); 2176 if (enum_val_base) { 2177 if (enum_val.value != enum_val_base->value) 2178 return "values differ for enum: " + enum_val.name; 2179 } 2180 } 2181 } 2182 return ""; 2183} 2184 2185} // namespace flatbuffers 2186