idl_parser.cpp revision 7cc72e4b11df064391a909ee7fa0ee5dd7630f9b
1/* 2 * Copyright 2014 Google Inc. All rights reserved. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include <algorithm> 18#include <list> 19#include <iostream> 20 21#ifdef _WIN32 22#if !defined(_USE_MATH_DEFINES) 23#define _USE_MATH_DEFINES // For M_PI. 24#endif // !defined(_USE_MATH_DEFINES) 25#endif // _WIN32 26 27#include <math.h> 28 29#include "flatbuffers/idl.h" 30#include "flatbuffers/util.h" 31 32namespace flatbuffers { 33 34const char *const kTypeNames[] = { 35 #define FLATBUFFERS_TD(ENUM, IDLTYPE, ALIASTYPE, \ 36 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \ 37 IDLTYPE, 38 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) 39 #undef FLATBUFFERS_TD 40 nullptr 41}; 42 43const char kTypeSizes[] = { 44 #define FLATBUFFERS_TD(ENUM, IDLTYPE, ALIASTYPE, \ 45 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \ 46 sizeof(CTYPE), 47 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) 48 #undef FLATBUFFERS_TD 49}; 50 51// The enums in the reflection schema should match the ones we use internally. 52// Compare the last element to check if these go out of sync. 53static_assert(BASE_TYPE_UNION == 54 static_cast<BaseType>(reflection::Union), 55 "enums don't match"); 56 57// Any parsing calls have to be wrapped in this macro, which automates 58// handling of recursive error checking a bit. It will check the received 59// CheckedError object, and return straight away on error. 60#define ECHECK(call) { auto ce = (call); if (ce.Check()) return ce; } 61 62// These two functions are called hundreds of times below, so define a short 63// form: 64#define NEXT() ECHECK(Next()) 65#define EXPECT(tok) ECHECK(Expect(tok)) 66 67static bool ValidateUTF8(const std::string &str) { 68 const char *s = &str[0]; 69 const char * const sEnd = s + str.length(); 70 while (s < sEnd) { 71 if (FromUTF8(&s) < 0) { 72 return false; 73 } 74 } 75 return true; 76} 77 78CheckedError Parser::Error(const std::string &msg) { 79 error_ = file_being_parsed_.length() ? AbsolutePath(file_being_parsed_) : ""; 80 #ifdef _WIN32 81 error_ += "(" + NumToString(line_) + ")"; // MSVC alike 82 #else 83 if (file_being_parsed_.length()) error_ += ":"; 84 error_ += NumToString(line_) + ":0"; // gcc alike 85 #endif 86 error_ += ": error: " + msg; 87 return CheckedError(true); 88} 89 90inline CheckedError NoError() { return CheckedError(false); } 91 92inline std::string OutOfRangeErrorMsg(int64_t val, const std::string &op, 93 int64_t limit) { 94 const std::string cause = NumToString(val) + op + NumToString(limit); 95 return "constant does not fit (" + cause + ")"; 96} 97 98// Ensure that integer values we parse fit inside the declared integer type. 99CheckedError Parser::CheckInRange(int64_t val, int64_t min, int64_t max) { 100 if (val < min) 101 return Error(OutOfRangeErrorMsg(val, " < ", min)); 102 else if (val > max) 103 return Error(OutOfRangeErrorMsg(val, " > ", max)); 104 else 105 return NoError(); 106} 107 108// atot: templated version of atoi/atof: convert a string to an instance of T. 109template<typename T> inline CheckedError atot(const char *s, Parser &parser, 110 T *val) { 111 int64_t i = StringToInt(s); 112 const int64_t min = flatbuffers::numeric_limits<T>::min(); 113 const int64_t max = flatbuffers::numeric_limits<T>::max(); 114 ECHECK(parser.CheckInRange(i, min, max)); 115 *val = (T)i; 116 return NoError(); 117} 118template<> inline CheckedError atot<uint64_t>(const char *s, Parser &parser, 119 uint64_t *val) { 120 (void)parser; 121 *val = StringToUInt(s); 122 return NoError(); 123} 124template<> inline CheckedError atot<bool>(const char *s, Parser &parser, 125 bool *val) { 126 (void)parser; 127 *val = 0 != atoi(s); 128 return NoError(); 129} 130template<> inline CheckedError atot<float>(const char *s, Parser &parser, 131 float *val) { 132 (void)parser; 133 *val = static_cast<float>(strtod(s, nullptr)); 134 return NoError(); 135} 136template<> inline CheckedError atot<double>(const char *s, Parser &parser, 137 double *val) { 138 (void)parser; 139 *val = strtod(s, nullptr); 140 return NoError(); 141} 142 143template<> inline CheckedError atot<Offset<void>>(const char *s, Parser &parser, 144 Offset<void> *val) { 145 (void)parser; 146 *val = Offset<void>(atoi(s)); 147 return NoError(); 148} 149 150std::string Namespace::GetFullyQualifiedName(const std::string &name, 151 size_t max_components) const { 152 // Early exit if we don't have a defined namespace. 153 if (components.size() == 0 || !max_components) { 154 return name; 155 } 156 std::stringstream stream; 157 for (size_t i = 0; i < std::min(components.size(), max_components); 158 i++) { 159 if (i) { 160 stream << "."; 161 } 162 stream << components[i]; 163 } 164 if (name.length()) stream << "." << name; 165 return stream.str(); 166} 167 168 169 170// Declare tokens we'll use. Single character tokens are represented by their 171// ascii character code (e.g. '{'), others above 256. 172#define FLATBUFFERS_GEN_TOKENS(TD) \ 173 TD(Eof, 256, "end of file") \ 174 TD(StringConstant, 257, "string constant") \ 175 TD(IntegerConstant, 258, "integer constant") \ 176 TD(FloatConstant, 259, "float constant") \ 177 TD(Identifier, 260, "identifier") \ 178 TD(Table, 261, "table") \ 179 TD(Struct, 262, "struct") \ 180 TD(Enum, 263, "enum") \ 181 TD(Union, 264, "union") \ 182 TD(NameSpace, 265, "namespace") \ 183 TD(RootType, 266, "root_type") \ 184 TD(FileIdentifier, 267, "file_identifier") \ 185 TD(FileExtension, 268, "file_extension") \ 186 TD(Include, 269, "include") \ 187 TD(Attribute, 270, "attribute") \ 188 TD(Null, 271, "null") \ 189 TD(Service, 272, "rpc_service") \ 190 TD(NativeInclude, 273, "native_include") \ 191 TD(BooleanConstant, 274, "boolean constant") 192#ifdef __GNUC__ 193__extension__ // Stop GCC complaining about trailing comma with -Wpendantic. 194#endif 195enum { 196 #define FLATBUFFERS_TOKEN(NAME, VALUE, STRING) kToken ## NAME = VALUE, 197 FLATBUFFERS_GEN_TOKENS(FLATBUFFERS_TOKEN) 198 #undef FLATBUFFERS_TOKEN 199 #define FLATBUFFERS_TD(ENUM, IDLTYPE, ALIASTYPE, \ 200 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \ 201 kToken ## ENUM, 202 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) 203 #undef FLATBUFFERS_TD 204}; 205 206static std::string TokenToString(int t) { 207 static const char *tokens[] = { 208 #define FLATBUFFERS_TOKEN(NAME, VALUE, STRING) STRING, 209 FLATBUFFERS_GEN_TOKENS(FLATBUFFERS_TOKEN) 210 #undef FLATBUFFERS_TOKEN 211 #define FLATBUFFERS_TD(ENUM, IDLTYPE, ALIASTYPE, \ 212 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \ 213 IDLTYPE, 214 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) 215 #undef FLATBUFFERS_TD 216 }; 217 if (t < 256) { // A single ascii char token. 218 std::string s; 219 s.append(1, static_cast<char>(t)); 220 return s; 221 } else { // Other tokens. 222 return tokens[t - 256]; 223 } 224} 225 226std::string Parser::TokenToStringId(int t) { 227 return TokenToString(t) + (t == kTokenIdentifier ? ": " + attribute_ : ""); 228} 229 230// Parses exactly nibbles worth of hex digits into a number, or error. 231CheckedError Parser::ParseHexNum(int nibbles, uint64_t *val) { 232 for (int i = 0; i < nibbles; i++) 233 if (!isxdigit(static_cast<const unsigned char>(cursor_[i]))) 234 return Error("escape code must be followed by " + NumToString(nibbles) + 235 " hex digits"); 236 std::string target(cursor_, cursor_ + nibbles); 237 *val = StringToUInt(target.c_str(), nullptr, 16); 238 cursor_ += nibbles; 239 return NoError(); 240} 241 242CheckedError Parser::SkipByteOrderMark() { 243 if (static_cast<unsigned char>(*cursor_) != 0xef) return NoError(); 244 cursor_++; 245 if (static_cast<unsigned char>(*cursor_) != 0xbb) return Error("invalid utf-8 byte order mark"); 246 cursor_++; 247 if (static_cast<unsigned char>(*cursor_) != 0xbf) return Error("invalid utf-8 byte order mark"); 248 cursor_++; 249 return NoError(); 250} 251 252bool IsIdentifierStart(char c) { 253 return isalpha(static_cast<unsigned char>(c)) || c == '_'; 254} 255 256CheckedError Parser::Next() { 257 doc_comment_.clear(); 258 bool seen_newline = false; 259 attribute_.clear(); 260 for (;;) { 261 char c = *cursor_++; 262 token_ = c; 263 switch (c) { 264 case '\0': cursor_--; token_ = kTokenEof; return NoError(); 265 case ' ': case '\r': case '\t': break; 266 case '\n': line_++; seen_newline = true; break; 267 case '{': case '}': case '(': case ')': case '[': case ']': 268 case ',': case ':': case ';': case '=': return NoError(); 269 case '.': 270 if(!isdigit(static_cast<const unsigned char>(*cursor_))) return NoError(); 271 return Error("floating point constant can\'t start with \".\""); 272 case '\"': 273 case '\'': { 274 int unicode_high_surrogate = -1; 275 276 while (*cursor_ != c) { 277 if (*cursor_ < ' ' && *cursor_ >= 0) 278 return Error("illegal character in string constant"); 279 if (*cursor_ == '\\') { 280 cursor_++; 281 if (unicode_high_surrogate != -1 && 282 *cursor_ != 'u') { 283 return Error( 284 "illegal Unicode sequence (unpaired high surrogate)"); 285 } 286 switch (*cursor_) { 287 case 'n': attribute_ += '\n'; cursor_++; break; 288 case 't': attribute_ += '\t'; cursor_++; break; 289 case 'r': attribute_ += '\r'; cursor_++; break; 290 case 'b': attribute_ += '\b'; cursor_++; break; 291 case 'f': attribute_ += '\f'; cursor_++; break; 292 case '\"': attribute_ += '\"'; cursor_++; break; 293 case '\'': attribute_ += '\''; cursor_++; break; 294 case '\\': attribute_ += '\\'; cursor_++; break; 295 case '/': attribute_ += '/'; cursor_++; break; 296 case 'x': { // Not in the JSON standard 297 cursor_++; 298 uint64_t val; 299 ECHECK(ParseHexNum(2, &val)); 300 attribute_ += static_cast<char>(val); 301 break; 302 } 303 case 'u': { 304 cursor_++; 305 uint64_t val; 306 ECHECK(ParseHexNum(4, &val)); 307 if (val >= 0xD800 && val <= 0xDBFF) { 308 if (unicode_high_surrogate != -1) { 309 return Error( 310 "illegal Unicode sequence (multiple high surrogates)"); 311 } else { 312 unicode_high_surrogate = static_cast<int>(val); 313 } 314 } else if (val >= 0xDC00 && val <= 0xDFFF) { 315 if (unicode_high_surrogate == -1) { 316 return Error( 317 "illegal Unicode sequence (unpaired low surrogate)"); 318 } else { 319 int code_point = 0x10000 + 320 ((unicode_high_surrogate & 0x03FF) << 10) + 321 (val & 0x03FF); 322 ToUTF8(code_point, &attribute_); 323 unicode_high_surrogate = -1; 324 } 325 } else { 326 if (unicode_high_surrogate != -1) { 327 return Error( 328 "illegal Unicode sequence (unpaired high surrogate)"); 329 } 330 ToUTF8(static_cast<int>(val), &attribute_); 331 } 332 break; 333 } 334 default: return Error("unknown escape code in string constant"); 335 } 336 } else { // printable chars + UTF-8 bytes 337 if (unicode_high_surrogate != -1) { 338 return Error( 339 "illegal Unicode sequence (unpaired high surrogate)"); 340 } 341 attribute_ += *cursor_++; 342 } 343 } 344 if (unicode_high_surrogate != -1) { 345 return Error( 346 "illegal Unicode sequence (unpaired high surrogate)"); 347 } 348 cursor_++; 349 if (!opts.allow_non_utf8 && !ValidateUTF8(attribute_)) { 350 return Error("illegal UTF-8 sequence"); 351 } 352 token_ = kTokenStringConstant; 353 return NoError(); 354 } 355 case '/': 356 if (*cursor_ == '/') { 357 const char *start = ++cursor_; 358 while (*cursor_ && *cursor_ != '\n' && *cursor_ != '\r') cursor_++; 359 if (*start == '/') { // documentation comment 360 if (cursor_ != source_ && !seen_newline) 361 return Error( 362 "a documentation comment should be on a line on its own"); 363 doc_comment_.push_back(std::string(start + 1, cursor_)); 364 } 365 break; 366 } else if (*cursor_ == '*') { 367 cursor_++; 368 // TODO: make nested. 369 while (*cursor_ != '*' || cursor_[1] != '/') { 370 if (*cursor_ == '\n') line_++; 371 if (!*cursor_) return Error("end of file in comment"); 372 cursor_++; 373 } 374 cursor_ += 2; 375 break; 376 } 377 // fall thru 378 default: 379 if (IsIdentifierStart(c)) { 380 // Collect all chars of an identifier: 381 const char *start = cursor_ - 1; 382 while (isalnum(static_cast<unsigned char>(*cursor_)) || 383 *cursor_ == '_') 384 cursor_++; 385 attribute_.append(start, cursor_); 386 // First, see if it is a type keyword from the table of types: 387 #define FLATBUFFERS_TD(ENUM, IDLTYPE, ALIASTYPE, \ 388 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \ 389 if (attribute_ == IDLTYPE || attribute_ == ALIASTYPE) { \ 390 token_ = kToken ## ENUM; \ 391 return NoError(); \ 392 } 393 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) 394 #undef FLATBUFFERS_TD 395 // If it's a boolean constant keyword, turn those into integers, 396 // which simplifies our logic downstream. 397 if (attribute_ == "true" || attribute_ == "false") { 398 attribute_ = NumToString(attribute_ == "true"); 399 token_ = kTokenBooleanConstant; 400 return NoError(); 401 } 402 // Check for declaration keywords: 403 if (attribute_ == "table") { 404 token_ = kTokenTable; 405 return NoError(); 406 } 407 if (attribute_ == "struct") { 408 token_ = kTokenStruct; 409 return NoError(); 410 } 411 if (attribute_ == "enum") { 412 token_ = kTokenEnum; 413 return NoError(); 414 } 415 if (attribute_ == "union") { 416 token_ = kTokenUnion; 417 return NoError(); 418 } 419 if (attribute_ == "namespace") { 420 token_ = kTokenNameSpace; 421 return NoError(); 422 } 423 if (attribute_ == "root_type") { 424 token_ = kTokenRootType; 425 return NoError(); 426 } 427 if (attribute_ == "include") { 428 token_ = kTokenInclude; 429 return NoError(); 430 } 431 if (attribute_ == "attribute") { 432 token_ = kTokenAttribute; 433 return NoError(); 434 } 435 if (attribute_ == "file_identifier") { 436 token_ = kTokenFileIdentifier; 437 return NoError(); 438 } 439 if (attribute_ == "file_extension") { 440 token_ = kTokenFileExtension; 441 return NoError(); 442 } 443 if (attribute_ == "null") { 444 token_ = kTokenNull; 445 return NoError(); 446 } 447 if (attribute_ == "rpc_service") { 448 token_ = kTokenService; 449 return NoError(); 450 } 451 if (attribute_ == "native_include") { 452 token_ = kTokenNativeInclude; 453 return NoError(); 454 } 455 // If not, it is a user-defined identifier: 456 token_ = kTokenIdentifier; 457 return NoError(); 458 } else if (isdigit(static_cast<unsigned char>(c)) || c == '-') { 459 const char *start = cursor_ - 1; 460 if (c == '-' && *cursor_ == '0' && 461 (cursor_[1] == 'x' || cursor_[1] == 'X')) { 462 ++start; 463 ++cursor_; 464 attribute_.append(&c, &c + 1); 465 c = '0'; 466 } 467 if (c == '0' && (*cursor_ == 'x' || *cursor_ == 'X')) { 468 cursor_++; 469 while (isxdigit(static_cast<unsigned char>(*cursor_))) cursor_++; 470 attribute_.append(start + 2, cursor_); 471 attribute_ = NumToString(static_cast<int64_t>( 472 StringToUInt(attribute_.c_str(), nullptr, 16))); 473 token_ = kTokenIntegerConstant; 474 return NoError(); 475 } 476 while (isdigit(static_cast<unsigned char>(*cursor_))) cursor_++; 477 if (*cursor_ == '.' || *cursor_ == 'e' || *cursor_ == 'E') { 478 if (*cursor_ == '.') { 479 cursor_++; 480 while (isdigit(static_cast<unsigned char>(*cursor_))) cursor_++; 481 } 482 // See if this float has a scientific notation suffix. Both JSON 483 // and C++ (through strtod() we use) have the same format: 484 if (*cursor_ == 'e' || *cursor_ == 'E') { 485 cursor_++; 486 if (*cursor_ == '+' || *cursor_ == '-') cursor_++; 487 while (isdigit(static_cast<unsigned char>(*cursor_))) cursor_++; 488 } 489 token_ = kTokenFloatConstant; 490 } else { 491 token_ = kTokenIntegerConstant; 492 } 493 attribute_.append(start, cursor_); 494 return NoError(); 495 } 496 std::string ch; 497 ch = c; 498 if (c < ' ' || c > '~') ch = "code: " + NumToString(c); 499 return Error("illegal character: " + ch); 500 } 501 } 502} 503 504// Check if a given token is next. 505bool Parser::Is(int t) { 506 return t == token_; 507} 508 509// Expect a given token to be next, consume it, or error if not present. 510CheckedError Parser::Expect(int t) { 511 if (t != token_) { 512 return Error("expecting: " + TokenToString(t) + " instead got: " + 513 TokenToStringId(token_)); 514 } 515 NEXT(); 516 return NoError(); 517} 518 519CheckedError Parser::ParseNamespacing(std::string *id, std::string *last) { 520 while (Is('.')) { 521 NEXT(); 522 *id += "."; 523 *id += attribute_; 524 if (last) *last = attribute_; 525 EXPECT(kTokenIdentifier); 526 } 527 return NoError(); 528} 529 530EnumDef *Parser::LookupEnum(const std::string &id) { 531 // Search thru parent namespaces. 532 for (int components = static_cast<int>(namespaces_.back()->components.size()); 533 components >= 0; components--) { 534 auto ed = enums_.Lookup( 535 namespaces_.back()->GetFullyQualifiedName(id, components)); 536 if (ed) return ed; 537 } 538 return nullptr; 539} 540 541CheckedError Parser::ParseTypeIdent(Type &type) { 542 std::string id = attribute_; 543 EXPECT(kTokenIdentifier); 544 ECHECK(ParseNamespacing(&id, nullptr)); 545 auto enum_def = LookupEnum(id); 546 if (enum_def) { 547 type = enum_def->underlying_type; 548 if (enum_def->is_union) type.base_type = BASE_TYPE_UNION; 549 } else { 550 type.base_type = BASE_TYPE_STRUCT; 551 type.struct_def = LookupCreateStruct(id); 552 } 553 return NoError(); 554} 555 556// Parse any IDL type. 557CheckedError Parser::ParseType(Type &type) { 558 if (token_ >= kTokenBOOL && token_ <= kTokenSTRING) { 559 type.base_type = static_cast<BaseType>(token_ - kTokenNONE); 560 NEXT(); 561 } else { 562 if (token_ == kTokenIdentifier) { 563 ECHECK(ParseTypeIdent(type)); 564 } else if (token_ == '[') { 565 NEXT(); 566 Type subtype; 567 ECHECK(ParseType(subtype)); 568 if (subtype.base_type == BASE_TYPE_VECTOR) { 569 // We could support this, but it will complicate things, and it's 570 // easier to work around with a struct around the inner vector. 571 return Error( 572 "nested vector types not supported (wrap in table first)."); 573 } 574 type = Type(BASE_TYPE_VECTOR, subtype.struct_def, subtype.enum_def); 575 type.element = subtype.base_type; 576 EXPECT(']'); 577 } else { 578 return Error("illegal type syntax"); 579 } 580 } 581 return NoError(); 582} 583 584CheckedError Parser::AddField(StructDef &struct_def, const std::string &name, 585 const Type &type, FieldDef **dest) { 586 auto &field = *new FieldDef(); 587 field.value.offset = 588 FieldIndexToOffset(static_cast<voffset_t>(struct_def.fields.vec.size())); 589 field.name = name; 590 field.file = struct_def.file; 591 field.value.type = type; 592 if (struct_def.fixed) { // statically compute the field offset 593 auto size = InlineSize(type); 594 auto alignment = InlineAlignment(type); 595 // structs_ need to have a predictable format, so we need to align to 596 // the largest scalar 597 struct_def.minalign = std::max(struct_def.minalign, alignment); 598 struct_def.PadLastField(alignment); 599 field.value.offset = static_cast<voffset_t>(struct_def.bytesize); 600 struct_def.bytesize += size; 601 } 602 if (struct_def.fields.Add(name, &field)) 603 return Error("field already exists: " + name); 604 *dest = &field; 605 return NoError(); 606} 607 608CheckedError Parser::ParseField(StructDef &struct_def) { 609 std::string name = attribute_; 610 611 if (structs_.Lookup(name)) 612 return Error("field name can not be the same as table/struct name"); 613 614 std::vector<std::string> dc = doc_comment_; 615 EXPECT(kTokenIdentifier); 616 EXPECT(':'); 617 Type type; 618 ECHECK(ParseType(type)); 619 620 if (struct_def.fixed && !IsScalar(type.base_type) && !IsStruct(type)) 621 return Error("structs_ may contain only scalar or struct fields"); 622 623 FieldDef *typefield = nullptr; 624 if (type.base_type == BASE_TYPE_UNION) { 625 // For union fields, add a second auto-generated field to hold the type, 626 // with a special suffix. 627 ECHECK(AddField(struct_def, name + UnionTypeFieldSuffix(), 628 type.enum_def->underlying_type, &typefield)); 629 } else if (type.base_type == BASE_TYPE_VECTOR && 630 type.element == BASE_TYPE_UNION) { 631 // Only cpp supports the union vector feature so far. 632 if (opts.lang_to_generate != IDLOptions::kCpp) { 633 return Error("Vectors of unions are not yet supported in all " 634 "the specified programming languages."); 635 } 636 // For vector of union fields, add a second auto-generated vector field to 637 // hold the types, with a special suffix. 638 Type union_vector(BASE_TYPE_VECTOR, nullptr, type.enum_def); 639 union_vector.element = BASE_TYPE_UTYPE; 640 ECHECK(AddField(struct_def, name + UnionTypeFieldSuffix(), 641 union_vector, &typefield)); 642 } 643 644 FieldDef *field; 645 ECHECK(AddField(struct_def, name, type, &field)); 646 647 if (token_ == '=') { 648 NEXT(); 649 if (!IsScalar(type.base_type)) 650 return Error("default values currently only supported for scalars"); 651 ECHECK(ParseSingleValue(field->value)); 652 } 653 if (IsFloat(field->value.type.base_type)) { 654 if (!strpbrk(field->value.constant.c_str(), ".eE")) 655 field->value.constant += ".0"; 656 } 657 658 if (type.enum_def && 659 IsScalar(type.base_type) && 660 !struct_def.fixed && 661 !type.enum_def->attributes.Lookup("bit_flags") && 662 !type.enum_def->ReverseLookup(static_cast<int>( 663 StringToInt(field->value.constant.c_str())))) 664 return Error("enum " + type.enum_def->name + 665 " does not have a declaration for this field\'s default of " + 666 field->value.constant); 667 668 field->doc_comment = dc; 669 ECHECK(ParseMetaData(&field->attributes)); 670 field->deprecated = field->attributes.Lookup("deprecated") != nullptr; 671 auto hash_name = field->attributes.Lookup("hash"); 672 if (hash_name) { 673 switch (type.base_type) { 674 case BASE_TYPE_INT: 675 case BASE_TYPE_UINT: { 676 if (FindHashFunction32(hash_name->constant.c_str()) == nullptr) 677 return Error("Unknown hashing algorithm for 32 bit types: " + 678 hash_name->constant); 679 break; 680 } 681 case BASE_TYPE_LONG: 682 case BASE_TYPE_ULONG: { 683 if (FindHashFunction64(hash_name->constant.c_str()) == nullptr) 684 return Error("Unknown hashing algorithm for 64 bit types: " + 685 hash_name->constant); 686 break; 687 } 688 default: 689 return Error( 690 "only int, uint, long and ulong data types support hashing."); 691 } 692 } 693 auto cpp_type = field->attributes.Lookup("cpp_type"); 694 if (cpp_type) { 695 if (!hash_name) 696 return Error("cpp_type can only be used with a hashed field"); 697 } 698 if (field->deprecated && struct_def.fixed) 699 return Error("can't deprecate fields in a struct"); 700 field->required = field->attributes.Lookup("required") != nullptr; 701 if (field->required && (struct_def.fixed || 702 IsScalar(field->value.type.base_type))) 703 return Error("only non-scalar fields in tables may be 'required'"); 704 field->key = field->attributes.Lookup("key") != nullptr; 705 if (field->key) { 706 if (struct_def.has_key) 707 return Error("only one field may be set as 'key'"); 708 struct_def.has_key = true; 709 if (!IsScalar(field->value.type.base_type)) { 710 field->required = true; 711 if (field->value.type.base_type != BASE_TYPE_STRING) 712 return Error("'key' field must be string or scalar type"); 713 } 714 } 715 716 field->native_inline = field->attributes.Lookup("native_inline") != nullptr; 717 if (field->native_inline && !IsStruct(field->value.type)) 718 return Error("native_inline can only be defined on structs'"); 719 720 auto nested = field->attributes.Lookup("nested_flatbuffer"); 721 if (nested) { 722 if (nested->type.base_type != BASE_TYPE_STRING) 723 return Error( 724 "nested_flatbuffer attribute must be a string (the root type)"); 725 if (field->value.type.base_type != BASE_TYPE_VECTOR || 726 field->value.type.element != BASE_TYPE_UCHAR) 727 return Error( 728 "nested_flatbuffer attribute may only apply to a vector of ubyte"); 729 // This will cause an error if the root type of the nested flatbuffer 730 // wasn't defined elsewhere. 731 LookupCreateStruct(nested->constant); 732 733 // Keep a pointer to StructDef in FieldDef to simplify re-use later 734 auto nested_qualified_name = namespaces_.back()->GetFullyQualifiedName(nested->constant); 735 field->nested_flatbuffer = structs_.Lookup(nested_qualified_name); 736 } 737 738 if (field->attributes.Lookup("flexbuffer")) { 739 field->flexbuffer = true; 740 uses_flexbuffers_ = true; 741 if (field->value.type.base_type != BASE_TYPE_VECTOR || 742 field->value.type.element != BASE_TYPE_UCHAR) 743 return Error( 744 "flexbuffer attribute may only apply to a vector of ubyte"); 745 } 746 747 if (typefield) { 748 if (!IsScalar(typefield->value.type.base_type)) { 749 // this is a union vector field 750 typefield->required = field->required; 751 } 752 // If this field is a union, and it has a manually assigned id, 753 // the automatically added type field should have an id as well (of N - 1). 754 auto attr = field->attributes.Lookup("id"); 755 if (attr) { 756 auto id = atoi(attr->constant.c_str()); 757 auto val = new Value(); 758 val->type = attr->type; 759 val->constant = NumToString(id - 1); 760 typefield->attributes.Add("id", val); 761 } 762 } 763 764 EXPECT(';'); 765 return NoError(); 766} 767 768CheckedError Parser::ParseString(Value &val) { 769 auto s = attribute_; 770 EXPECT(kTokenStringConstant); 771 val.constant = NumToString(builder_.CreateString(s).o); 772 return NoError(); 773} 774 775CheckedError Parser::ParseComma() { 776 if (!opts.protobuf_ascii_alike) EXPECT(','); 777 return NoError(); 778} 779 780CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field, 781 size_t parent_fieldn, 782 const StructDef *parent_struct_def) { 783 switch (val.type.base_type) { 784 case BASE_TYPE_UNION: { 785 assert(field); 786 std::string constant; 787 // Find corresponding type field we may have already parsed. 788 for (auto elem = field_stack_.rbegin(); 789 elem != field_stack_.rbegin() + parent_fieldn; ++elem) { 790 auto &type = elem->second->value.type; 791 if (type.base_type == BASE_TYPE_UTYPE && 792 type.enum_def == val.type.enum_def) { 793 constant = elem->first.constant; 794 break; 795 } 796 } 797 if (constant.empty()) { 798 // We haven't seen the type field yet. Sadly a lot of JSON writers 799 // output these in alphabetical order, meaning it comes after this 800 // value. So we scan past the value to find it, then come back here. 801 auto type_name = field->name + UnionTypeFieldSuffix(); 802 assert(parent_struct_def); 803 auto type_field = parent_struct_def->fields.Lookup(type_name); 804 assert(type_field); // Guaranteed by ParseField(). 805 // Remember where we are in the source file, so we can come back here. 806 auto backup = *static_cast<ParserState *>(this); 807 ECHECK(SkipAnyJsonValue()); // The table. 808 ECHECK(ParseComma()); 809 auto next_name = attribute_; 810 if (Is(kTokenStringConstant)) { 811 NEXT(); 812 } else { 813 EXPECT(kTokenIdentifier); 814 } 815 if (next_name != type_name) 816 return Error("missing type field after this union value: " + 817 type_name); 818 EXPECT(':'); 819 Value type_val = type_field->value; 820 ECHECK(ParseAnyValue(type_val, type_field, 0, nullptr)); 821 constant = type_val.constant; 822 // Got the information we needed, now rewind: 823 *static_cast<ParserState *>(this) = backup; 824 } 825 uint8_t enum_idx; 826 ECHECK(atot(constant.c_str(), *this, &enum_idx)); 827 auto enum_val = val.type.enum_def->ReverseLookup(enum_idx); 828 if (!enum_val) return Error("illegal type id for: " + field->name); 829 if (enum_val->union_type.base_type == BASE_TYPE_STRUCT) { 830 ECHECK(ParseTable(*enum_val->union_type.struct_def, &val.constant, 831 nullptr)); 832 if (enum_val->union_type.struct_def->fixed) { 833 // All BASE_TYPE_UNION values are offsets, so turn this into one. 834 SerializeStruct(*enum_val->union_type.struct_def, val); 835 builder_.ClearOffsets(); 836 val.constant = NumToString(builder_.GetSize()); 837 } 838 } else if (enum_val->union_type.base_type == BASE_TYPE_STRING) { 839 ECHECK(ParseString(val)); 840 } else { 841 assert(false); 842 } 843 break; 844 } 845 case BASE_TYPE_STRUCT: 846 ECHECK(ParseTable(*val.type.struct_def, &val.constant, nullptr)); 847 break; 848 case BASE_TYPE_STRING: { 849 ECHECK(ParseString(val)); 850 break; 851 } 852 case BASE_TYPE_VECTOR: { 853 uoffset_t off; 854 ECHECK(ParseVector(val.type.VectorType(), &off)); 855 val.constant = NumToString(off); 856 break; 857 } 858 case BASE_TYPE_INT: 859 case BASE_TYPE_UINT: 860 case BASE_TYPE_LONG: 861 case BASE_TYPE_ULONG: { 862 if (field && field->attributes.Lookup("hash") && 863 (token_ == kTokenIdentifier || token_ == kTokenStringConstant)) { 864 ECHECK(ParseHash(val, field)); 865 } else { 866 ECHECK(ParseSingleValue(val)); 867 } 868 break; 869 } 870 default: 871 ECHECK(ParseSingleValue(val)); 872 break; 873 } 874 return NoError(); 875} 876 877void Parser::SerializeStruct(const StructDef &struct_def, const Value &val) { 878 assert(val.constant.length() == struct_def.bytesize); 879 builder_.Align(struct_def.minalign); 880 builder_.PushBytes(reinterpret_cast<const uint8_t *>(val.constant.c_str()), 881 struct_def.bytesize); 882 builder_.AddStructOffset(val.offset, builder_.GetSize()); 883} 884 885CheckedError Parser::ParseTableDelimiters(size_t &fieldn, 886 const StructDef *struct_def, 887 ParseTableDelimitersBody body, 888 void *state) { 889 // We allow tables both as JSON object{ .. } with field names 890 // or vector[..] with all fields in order 891 char terminator = '}'; 892 bool is_nested_vector = struct_def && Is('['); 893 if (is_nested_vector) { 894 NEXT(); 895 terminator = ']'; 896 } else { 897 EXPECT('{'); 898 } 899 for (;;) { 900 if ((!opts.strict_json || !fieldn) && Is(terminator)) break; 901 std::string name; 902 if (is_nested_vector) { 903 if (fieldn > struct_def->fields.vec.size()) { 904 return Error("too many unnamed fields in nested array"); 905 } 906 name = struct_def->fields.vec[fieldn]->name; 907 } else { 908 name = attribute_; 909 if (Is(kTokenStringConstant)) { 910 NEXT(); 911 } else { 912 EXPECT(opts.strict_json ? kTokenStringConstant : kTokenIdentifier); 913 } 914 if (!opts.protobuf_ascii_alike || !(Is('{') || Is('['))) EXPECT(':'); 915 } 916 ECHECK(body(name, fieldn, struct_def, state)); 917 if (Is(terminator)) break; 918 ECHECK(ParseComma()); 919 } 920 NEXT(); 921 if (is_nested_vector && fieldn != struct_def->fields.vec.size()) { 922 return Error("wrong number of unnamed fields in table vector"); 923 } 924 return NoError(); 925} 926 927CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, 928 uoffset_t *ovalue) { 929 size_t fieldn_outer = 0; 930 auto err = ParseTableDelimiters(fieldn_outer, &struct_def, 931 [](const std::string &name, size_t &fieldn, 932 const StructDef *struct_def_inner, 933 void *state) -> CheckedError { 934 Parser *parser = static_cast<Parser *>(state); 935 if (name == "$schema") { 936 ECHECK(parser->Expect(kTokenStringConstant)); 937 return NoError(); 938 } 939 auto field = struct_def_inner->fields.Lookup(name); 940 if (!field) { 941 if (!parser->opts.skip_unexpected_fields_in_json) { 942 return parser->Error("unknown field: " + name); 943 } else { 944 ECHECK(parser->SkipAnyJsonValue()); 945 } 946 } else { 947 if (parser->Is(kTokenNull)) { 948 ECHECK(parser->Next()); // Ignore this field. 949 } else { 950 Value val = field->value; 951 if (field->flexbuffer) { 952 flexbuffers::Builder builder(1024, 953 flexbuffers::BUILDER_FLAG_SHARE_ALL); 954 ECHECK(parser->ParseFlexBufferValue(&builder)); 955 builder.Finish(); 956 auto off = parser->builder_.CreateVector(builder.GetBuffer()); 957 val.constant = NumToString(off.o); 958 } else if (field->nested_flatbuffer) { 959 ECHECK(parser->ParseNestedFlatbuffer(val, field, fieldn, struct_def_inner)); 960 } else { 961 ECHECK(parser->ParseAnyValue(val, field, fieldn, struct_def_inner)); 962 } 963 // Hardcoded insertion-sort with error-check. 964 // If fields are specified in order, then this loop exits immediately. 965 auto elem = parser->field_stack_.rbegin(); 966 for (; elem != parser->field_stack_.rbegin() + fieldn; ++elem) { 967 auto existing_field = elem->second; 968 if (existing_field == field) 969 return parser->Error("field set more than once: " + field->name); 970 if (existing_field->value.offset < field->value.offset) break; 971 } 972 // Note: elem points to before the insertion point, thus .base() points 973 // to the correct spot. 974 parser->field_stack_.insert(elem.base(), 975 std::make_pair(val, field)); 976 fieldn++; 977 } 978 } 979 return NoError(); 980 }, this); 981 ECHECK(err); 982 983 // Check if all required fields are parsed. 984 for (auto field_it = struct_def.fields.vec.begin(); 985 field_it != struct_def.fields.vec.end(); 986 ++field_it) { 987 auto required_field = *field_it; 988 if (!required_field->required) { 989 continue; 990 } 991 bool found = false; 992 for (auto pf_it = field_stack_.end() - fieldn_outer; 993 pf_it != field_stack_.end(); 994 ++pf_it) { 995 auto parsed_field = pf_it->second; 996 if (parsed_field == required_field) { 997 found = true; 998 break; 999 } 1000 } 1001 if (!found) { 1002 return Error("required field is missing: " + required_field->name + " in " + struct_def.name); 1003 } 1004 } 1005 1006 if (struct_def.fixed && fieldn_outer != struct_def.fields.vec.size()) 1007 return Error("struct: wrong number of initializers: " + struct_def.name); 1008 1009 auto start = struct_def.fixed 1010 ? builder_.StartStruct(struct_def.minalign) 1011 : builder_.StartTable(); 1012 1013 for (size_t size = struct_def.sortbysize ? sizeof(largest_scalar_t) : 1; 1014 size; 1015 size /= 2) { 1016 // Go through elements in reverse, since we're building the data backwards. 1017 for (auto it = field_stack_.rbegin(); it != field_stack_.rbegin() + 1018 fieldn_outer; 1019 ++it) { 1020 auto &field_value = it->first; 1021 auto field = it->second; 1022 if (!struct_def.sortbysize || 1023 size == SizeOf(field_value.type.base_type)) { 1024 switch (field_value.type.base_type) { 1025 #define FLATBUFFERS_TD(ENUM, IDLTYPE, ALIASTYPE, \ 1026 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \ 1027 case BASE_TYPE_ ## ENUM: \ 1028 builder_.Pad(field->padding); \ 1029 if (struct_def.fixed) { \ 1030 CTYPE val; \ 1031 ECHECK(atot(field_value.constant.c_str(), *this, &val)); \ 1032 builder_.PushElement(val); \ 1033 } else { \ 1034 CTYPE val, valdef; \ 1035 ECHECK(atot(field_value.constant.c_str(), *this, &val)); \ 1036 ECHECK(atot(field->value.constant.c_str(), *this, &valdef)); \ 1037 builder_.AddElement(field_value.offset, val, valdef); \ 1038 } \ 1039 break; 1040 FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD); 1041 #undef FLATBUFFERS_TD 1042 #define FLATBUFFERS_TD(ENUM, IDLTYPE, ALIASTYPE, \ 1043 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \ 1044 case BASE_TYPE_ ## ENUM: \ 1045 builder_.Pad(field->padding); \ 1046 if (IsStruct(field->value.type)) { \ 1047 SerializeStruct(*field->value.type.struct_def, field_value); \ 1048 } else { \ 1049 CTYPE val; \ 1050 ECHECK(atot(field_value.constant.c_str(), *this, &val)); \ 1051 builder_.AddOffset(field_value.offset, val); \ 1052 } \ 1053 break; 1054 FLATBUFFERS_GEN_TYPES_POINTER(FLATBUFFERS_TD); 1055 #undef FLATBUFFERS_TD 1056 } 1057 } 1058 } 1059 } 1060 for (size_t i = 0; i < fieldn_outer; i++) field_stack_.pop_back(); 1061 1062 if (struct_def.fixed) { 1063 builder_.ClearOffsets(); 1064 builder_.EndStruct(); 1065 assert(value); 1066 // Temporarily store this struct in the value string, since it is to 1067 // be serialized in-place elsewhere. 1068 value->assign( 1069 reinterpret_cast<const char *>(builder_.GetCurrentBufferPointer()), 1070 struct_def.bytesize); 1071 builder_.PopBytes(struct_def.bytesize); 1072 assert(!ovalue); 1073 } else { 1074 auto val = builder_.EndTable(start, 1075 static_cast<voffset_t>(struct_def.fields.vec.size())); 1076 if (ovalue) *ovalue = val; 1077 if (value) *value = NumToString(val); 1078 } 1079 return NoError(); 1080} 1081 1082CheckedError Parser::ParseVectorDelimiters(size_t &count, 1083 ParseVectorDelimitersBody body, 1084 void *state) { 1085 EXPECT('['); 1086 for (;;) { 1087 if ((!opts.strict_json || !count) && Is(']')) break; 1088 ECHECK(body(count, state)); 1089 count++; 1090 if (Is(']')) break; 1091 ECHECK(ParseComma()); 1092 } 1093 NEXT(); 1094 return NoError(); 1095} 1096 1097CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue) { 1098 size_t count = 0; 1099 std::pair<Parser *, const Type &> parser_and_type_state(this, type); 1100 auto err = ParseVectorDelimiters(count, 1101 [](size_t &, void *state) -> CheckedError { 1102 auto *parser_and_type = 1103 static_cast<std::pair<Parser *, const Type &> *>(state); 1104 auto *parser = parser_and_type->first; 1105 Value val; 1106 val.type = parser_and_type->second; 1107 ECHECK(parser->ParseAnyValue(val, nullptr, 0, nullptr)); 1108 parser->field_stack_.push_back(std::make_pair(val, nullptr)); 1109 return NoError(); 1110 }, &parser_and_type_state); 1111 ECHECK(err); 1112 1113 builder_.StartVector(count * InlineSize(type) / InlineAlignment(type), 1114 InlineAlignment(type)); 1115 for (size_t i = 0; i < count; i++) { 1116 // start at the back, since we're building the data backwards. 1117 auto &val = field_stack_.back().first; 1118 switch (val.type.base_type) { 1119 #define FLATBUFFERS_TD(ENUM, IDLTYPE, ALIASTYPE, \ 1120 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \ 1121 case BASE_TYPE_ ## ENUM: \ 1122 if (IsStruct(val.type)) SerializeStruct(*val.type.struct_def, val); \ 1123 else { \ 1124 CTYPE elem; \ 1125 ECHECK(atot(val.constant.c_str(), *this, &elem)); \ 1126 builder_.PushElement(elem); \ 1127 } \ 1128 break; 1129 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) 1130 #undef FLATBUFFERS_TD 1131 } 1132 field_stack_.pop_back(); 1133 } 1134 1135 builder_.ClearOffsets(); 1136 *ovalue = builder_.EndVector(count); 1137 return NoError(); 1138} 1139 1140CheckedError Parser::ParseNestedFlatbuffer(Value &val, FieldDef *field, 1141 size_t fieldn, 1142 const StructDef *parent_struct_def) { 1143 if (token_ == '[') {// backwards compat for 'legacy' ubyte buffers 1144 ECHECK(ParseAnyValue(val, field, fieldn, parent_struct_def)); 1145 } else { 1146 auto cursor_at_value_begin = cursor_; 1147 ECHECK(SkipAnyJsonValue()); 1148 std::string substring(cursor_at_value_begin -1 , cursor_ -1); 1149 1150 // Create and initialize new parser 1151 Parser nested_parser; 1152 assert(field->nested_flatbuffer); 1153 nested_parser.root_struct_def_ = field->nested_flatbuffer; 1154 nested_parser.enums_ = enums_; 1155 nested_parser.opts = opts; 1156 nested_parser.uses_flexbuffers_ = uses_flexbuffers_; 1157 1158 // Parse JSON substring into new flatbuffer builder using nested_parser 1159 if (!nested_parser.Parse(substring.c_str(), nullptr, nullptr)) { 1160 ECHECK(Error(nested_parser.error_)); 1161 } 1162 auto off = builder_.CreateVector(nested_parser.builder_.GetBufferPointer(), nested_parser.builder_.GetSize()); 1163 val.constant = NumToString(off.o); 1164 1165 // Clean nested_parser before destruction to avoid deleting the elements in the SymbolTables 1166 nested_parser.enums_.dict.clear(); 1167 nested_parser.enums_.vec.clear(); 1168 } 1169 return NoError(); 1170} 1171 1172CheckedError Parser::ParseMetaData(SymbolTable<Value> *attributes) { 1173 if (Is('(')) { 1174 NEXT(); 1175 for (;;) { 1176 auto name = attribute_; 1177 EXPECT(kTokenIdentifier); 1178 if (known_attributes_.find(name) == known_attributes_.end()) 1179 return Error("user define attributes must be declared before use: " + 1180 name); 1181 auto e = new Value(); 1182 attributes->Add(name, e); 1183 if (Is(':')) { 1184 NEXT(); 1185 ECHECK(ParseSingleValue(*e)); 1186 } 1187 if (Is(')')) { NEXT(); break; } 1188 EXPECT(','); 1189 } 1190 } 1191 return NoError(); 1192} 1193 1194CheckedError Parser::TryTypedValue(int dtoken, bool check, Value &e, 1195 BaseType req, bool *destmatch) { 1196 bool match = dtoken == token_; 1197 if (match) { 1198 *destmatch = true; 1199 e.constant = attribute_; 1200 if (!check) { 1201 if (e.type.base_type == BASE_TYPE_NONE) { 1202 e.type.base_type = req; 1203 } else { 1204 return Error(std::string("type mismatch: expecting: ") + 1205 kTypeNames[e.type.base_type] + 1206 ", found: " + 1207 kTypeNames[req]); 1208 } 1209 } 1210 NEXT(); 1211 } 1212 return NoError(); 1213} 1214 1215CheckedError Parser::ParseEnumFromString(Type &type, int64_t *result) { 1216 *result = 0; 1217 // Parse one or more enum identifiers, separated by spaces. 1218 const char *next = attribute_.c_str(); 1219 do { 1220 const char *divider = strchr(next, ' '); 1221 std::string word; 1222 if (divider) { 1223 word = std::string(next, divider); 1224 next = divider + strspn(divider, " "); 1225 } else { 1226 word = next; 1227 next += word.length(); 1228 } 1229 if (type.enum_def) { // The field has an enum type 1230 auto enum_val = type.enum_def->vals.Lookup(word); 1231 if (!enum_val) 1232 return Error("unknown enum value: " + word + 1233 ", for enum: " + type.enum_def->name); 1234 *result |= enum_val->value; 1235 } else { // No enum type, probably integral field. 1236 if (!IsInteger(type.base_type)) 1237 return Error("not a valid value for this field: " + word); 1238 // TODO: could check if its a valid number constant here. 1239 const char *dot = strrchr(word.c_str(), '.'); 1240 if (!dot) 1241 return Error("enum values need to be qualified by an enum type"); 1242 std::string enum_def_str(word.c_str(), dot); 1243 std::string enum_val_str(dot + 1, word.c_str() + word.length()); 1244 auto enum_def = LookupEnum(enum_def_str); 1245 if (!enum_def) return Error("unknown enum: " + enum_def_str); 1246 auto enum_val = enum_def->vals.Lookup(enum_val_str); 1247 if (!enum_val) return Error("unknown enum value: " + enum_val_str); 1248 *result |= enum_val->value; 1249 } 1250 } while(*next); 1251 return NoError(); 1252} 1253 1254 1255CheckedError Parser::ParseHash(Value &e, FieldDef* field) { 1256 assert(field); 1257 Value *hash_name = field->attributes.Lookup("hash"); 1258 switch (e.type.base_type) { 1259 case BASE_TYPE_INT: { 1260 auto hash = FindHashFunction32(hash_name->constant.c_str()); 1261 int32_t hashed_value = static_cast<int32_t>(hash(attribute_.c_str())); 1262 e.constant = NumToString(hashed_value); 1263 break; 1264 } 1265 case BASE_TYPE_UINT: { 1266 auto hash = FindHashFunction32(hash_name->constant.c_str()); 1267 uint32_t hashed_value = hash(attribute_.c_str()); 1268 e.constant = NumToString(hashed_value); 1269 break; 1270 } 1271 case BASE_TYPE_LONG: { 1272 auto hash = FindHashFunction64(hash_name->constant.c_str()); 1273 int64_t hashed_value = static_cast<int64_t>(hash(attribute_.c_str())); 1274 e.constant = NumToString(hashed_value); 1275 break; 1276 } 1277 case BASE_TYPE_ULONG: { 1278 auto hash = FindHashFunction64(hash_name->constant.c_str()); 1279 uint64_t hashed_value = hash(attribute_.c_str()); 1280 e.constant = NumToString(hashed_value); 1281 break; 1282 } 1283 default: 1284 assert(0); 1285 } 1286 NEXT(); 1287 return NoError(); 1288} 1289 1290CheckedError Parser::TokenError() { 1291 return Error("cannot parse value starting with: " + 1292 TokenToStringId(token_)); 1293} 1294 1295CheckedError Parser::ParseSingleValue(Value &e) { 1296 // First see if this could be a conversion function: 1297 if (token_ == kTokenIdentifier && *cursor_ == '(') { 1298 auto functionname = attribute_; 1299 NEXT(); 1300 EXPECT('('); 1301 ECHECK(ParseSingleValue(e)); 1302 EXPECT(')'); 1303 #define FLATBUFFERS_FN_DOUBLE(name, op) \ 1304 if (functionname == name) { \ 1305 auto x = strtod(e.constant.c_str(), nullptr); \ 1306 e.constant = NumToString(op); \ 1307 } 1308 FLATBUFFERS_FN_DOUBLE("deg", x / M_PI * 180); 1309 FLATBUFFERS_FN_DOUBLE("rad", x * M_PI / 180); 1310 FLATBUFFERS_FN_DOUBLE("sin", sin(x)); 1311 FLATBUFFERS_FN_DOUBLE("cos", cos(x)); 1312 FLATBUFFERS_FN_DOUBLE("tan", tan(x)); 1313 FLATBUFFERS_FN_DOUBLE("asin", asin(x)); 1314 FLATBUFFERS_FN_DOUBLE("acos", acos(x)); 1315 FLATBUFFERS_FN_DOUBLE("atan", atan(x)); 1316 // TODO(wvo): add more useful conversion functions here. 1317 #undef FLATBUFFERS_FN_DOUBLE 1318 // Then check if this could be a string/identifier enum value: 1319 } else if (e.type.base_type != BASE_TYPE_STRING && 1320 e.type.base_type != BASE_TYPE_NONE && 1321 (token_ == kTokenIdentifier || token_ == kTokenStringConstant)) { 1322 if (IsIdentifierStart(attribute_[0])) { // Enum value. 1323 int64_t val; 1324 ECHECK(ParseEnumFromString(e.type, &val)); 1325 e.constant = NumToString(val); 1326 NEXT(); 1327 } else { // Numeric constant in string. 1328 if (IsInteger(e.type.base_type)) { 1329 char *end; 1330 e.constant = NumToString(StringToInt(attribute_.c_str(), &end)); 1331 if (*end) 1332 return Error("invalid integer: " + attribute_); 1333 } else if (IsFloat(e.type.base_type)) { 1334 char *end; 1335 e.constant = NumToString(strtod(attribute_.c_str(), &end)); 1336 if (*end) 1337 return Error("invalid float: " + attribute_); 1338 } else { 1339 assert(0); // Shouldn't happen, we covered all types. 1340 e.constant = "0"; 1341 } 1342 NEXT(); 1343 } 1344 } else { 1345 bool match = false; 1346 ECHECK(TryTypedValue(kTokenIntegerConstant, 1347 IsScalar(e.type.base_type), 1348 e, 1349 BASE_TYPE_INT, 1350 &match)); 1351 ECHECK(TryTypedValue(kTokenBooleanConstant, 1352 IsBool(e.type.base_type), 1353 e, 1354 BASE_TYPE_BOOL, 1355 &match)); 1356 ECHECK(TryTypedValue(kTokenFloatConstant, 1357 IsFloat(e.type.base_type), 1358 e, 1359 BASE_TYPE_FLOAT, 1360 &match)); 1361 ECHECK(TryTypedValue(kTokenStringConstant, 1362 e.type.base_type == BASE_TYPE_STRING, 1363 e, 1364 BASE_TYPE_STRING, 1365 &match)); 1366 if (!match) return TokenError(); 1367 } 1368 return NoError(); 1369} 1370 1371StructDef *Parser::LookupCreateStruct(const std::string &name, 1372 bool create_if_new, bool definition) { 1373 std::string qualified_name = namespaces_.back()->GetFullyQualifiedName(name); 1374 // See if it exists pre-declared by an unqualified use. 1375 auto struct_def = structs_.Lookup(name); 1376 if (struct_def && struct_def->predecl) { 1377 if (definition) { 1378 // Make sure it has the current namespace, and is registered under its 1379 // qualified name. 1380 struct_def->defined_namespace = namespaces_.back(); 1381 structs_.Move(name, qualified_name); 1382 } 1383 return struct_def; 1384 } 1385 // See if it exists pre-declared by an qualified use. 1386 struct_def = structs_.Lookup(qualified_name); 1387 if (struct_def && struct_def->predecl) { 1388 if (definition) { 1389 // Make sure it has the current namespace. 1390 struct_def->defined_namespace = namespaces_.back(); 1391 } 1392 return struct_def; 1393 } 1394 if (!definition) { 1395 // Search thru parent namespaces. 1396 for (size_t components = namespaces_.back()->components.size(); 1397 components && !struct_def; components--) { 1398 struct_def = structs_.Lookup( 1399 namespaces_.back()->GetFullyQualifiedName(name, components - 1)); 1400 } 1401 } 1402 if (!struct_def && create_if_new) { 1403 struct_def = new StructDef(); 1404 if (definition) { 1405 structs_.Add(qualified_name, struct_def); 1406 struct_def->name = name; 1407 struct_def->defined_namespace = namespaces_.back(); 1408 } else { 1409 // Not a definition. 1410 // Rather than failing, we create a "pre declared" StructDef, due to 1411 // circular references, and check for errors at the end of parsing. 1412 // It is defined in the root namespace, since we don't know what the 1413 // final namespace will be. 1414 // TODO: maybe safer to use special namespace? 1415 structs_.Add(name, struct_def); 1416 struct_def->name = name; 1417 struct_def->defined_namespace = new Namespace(); 1418 namespaces_.insert(namespaces_.begin(), struct_def->defined_namespace); 1419 } 1420 } 1421 return struct_def; 1422} 1423 1424CheckedError Parser::ParseEnum(bool is_union, EnumDef **dest) { 1425 std::vector<std::string> enum_comment = doc_comment_; 1426 NEXT(); 1427 std::string enum_name = attribute_; 1428 EXPECT(kTokenIdentifier); 1429 auto &enum_def = *new EnumDef(); 1430 enum_def.name = enum_name; 1431 enum_def.file = file_being_parsed_; 1432 enum_def.doc_comment = enum_comment; 1433 enum_def.is_union = is_union; 1434 enum_def.defined_namespace = namespaces_.back(); 1435 if (enums_.Add(namespaces_.back()->GetFullyQualifiedName(enum_name), 1436 &enum_def)) 1437 return Error("enum already exists: " + enum_name); 1438 if (is_union) { 1439 enum_def.underlying_type.base_type = BASE_TYPE_UTYPE; 1440 enum_def.underlying_type.enum_def = &enum_def; 1441 } else { 1442 if (opts.proto_mode) { 1443 enum_def.underlying_type.base_type = BASE_TYPE_INT; 1444 } else { 1445 // Give specialized error message, since this type spec used to 1446 // be optional in the first FlatBuffers release. 1447 if (!Is(':')) { 1448 return Error("must specify the underlying integer type for this" 1449 " enum (e.g. \': short\', which was the default)."); 1450 } else { 1451 NEXT(); 1452 } 1453 // Specify the integer type underlying this enum. 1454 ECHECK(ParseType(enum_def.underlying_type)); 1455 if (!IsInteger(enum_def.underlying_type.base_type)) 1456 return Error("underlying enum type must be integral"); 1457 } 1458 // Make this type refer back to the enum it was derived from. 1459 enum_def.underlying_type.enum_def = &enum_def; 1460 } 1461 ECHECK(ParseMetaData(&enum_def.attributes)); 1462 EXPECT('{'); 1463 if (is_union) enum_def.vals.Add("NONE", new EnumVal("NONE", 0)); 1464 for (;;) { 1465 if (opts.proto_mode && attribute_ == "option") { 1466 ECHECK(ParseProtoOption()); 1467 } else { 1468 auto value_name = attribute_; 1469 auto full_name = value_name; 1470 std::vector<std::string> value_comment = doc_comment_; 1471 EXPECT(kTokenIdentifier); 1472 if (is_union) { 1473 ECHECK(ParseNamespacing(&full_name, &value_name)); 1474 if (opts.union_value_namespacing) { 1475 // Since we can't namespace the actual enum identifiers, turn 1476 // namespace parts into part of the identifier. 1477 value_name = full_name; 1478 std::replace(value_name.begin(), value_name.end(), '.', '_'); 1479 } 1480 } 1481 auto prevsize = enum_def.vals.vec.size(); 1482 auto value = enum_def.vals.vec.size() 1483 ? enum_def.vals.vec.back()->value + 1 1484 : 0; 1485 auto &ev = *new EnumVal(value_name, value); 1486 if (enum_def.vals.Add(value_name, &ev)) 1487 return Error("enum value already exists: " + value_name); 1488 ev.doc_comment = value_comment; 1489 if (is_union) { 1490 if (Is(':')) { 1491 NEXT(); 1492 ECHECK(ParseType(ev.union_type)); 1493 if (ev.union_type.base_type != BASE_TYPE_STRUCT && 1494 ev.union_type.base_type != BASE_TYPE_STRING) 1495 return Error("union value type may only be table/struct/string"); 1496 enum_def.uses_type_aliases = true; 1497 } else { 1498 ev.union_type = Type(BASE_TYPE_STRUCT, LookupCreateStruct(full_name)); 1499 } 1500 } 1501 if (Is('=')) { 1502 NEXT(); 1503 ev.value = StringToInt(attribute_.c_str()); 1504 EXPECT(kTokenIntegerConstant); 1505 if (!opts.proto_mode && prevsize && 1506 enum_def.vals.vec[prevsize - 1]->value >= ev.value) 1507 return Error("enum values must be specified in ascending order"); 1508 } 1509 if (is_union) { 1510 if (ev.value < 0 || ev.value >= 256) 1511 return Error("union enum value must fit in a ubyte"); 1512 } 1513 if (opts.proto_mode && Is('[')) { 1514 NEXT(); 1515 // ignore attributes on enums. 1516 while (token_ != ']') NEXT(); 1517 NEXT(); 1518 } 1519 } 1520 if (!Is(opts.proto_mode ? ';' : ',')) break; 1521 NEXT(); 1522 if (Is('}')) break; 1523 } 1524 EXPECT('}'); 1525 if (enum_def.attributes.Lookup("bit_flags")) { 1526 for (auto it = enum_def.vals.vec.begin(); it != enum_def.vals.vec.end(); 1527 ++it) { 1528 if (static_cast<size_t>((*it)->value) >= 1529 SizeOf(enum_def.underlying_type.base_type) * 8) 1530 return Error("bit flag out of range of underlying integral type"); 1531 (*it)->value = 1LL << (*it)->value; 1532 } 1533 } 1534 if (dest) *dest = &enum_def; 1535 types_.Add(namespaces_.back()->GetFullyQualifiedName(enum_def.name), 1536 new Type(BASE_TYPE_UNION, nullptr, &enum_def)); 1537 return NoError(); 1538} 1539 1540CheckedError Parser::StartStruct(const std::string &name, StructDef **dest) { 1541 auto &struct_def = *LookupCreateStruct(name, true, true); 1542 if (!struct_def.predecl) return Error("datatype already exists: " + name); 1543 struct_def.predecl = false; 1544 struct_def.name = name; 1545 struct_def.file = file_being_parsed_; 1546 // Move this struct to the back of the vector just in case it was predeclared, 1547 // to preserve declaration order. 1548 *std::remove(structs_.vec.begin(), structs_.vec.end(), &struct_def) = &struct_def; 1549 *dest = &struct_def; 1550 return NoError(); 1551} 1552 1553CheckedError Parser::CheckClash(std::vector<FieldDef*> &fields, 1554 StructDef *struct_def, 1555 const char *suffix, 1556 BaseType basetype) { 1557 auto len = strlen(suffix); 1558 for (auto it = fields.begin(); it != fields.end(); ++it) { 1559 auto &fname = (*it)->name; 1560 if (fname.length() > len && 1561 fname.compare(fname.length() - len, len, suffix) == 0 && 1562 (*it)->value.type.base_type != BASE_TYPE_UTYPE) { 1563 auto field = struct_def->fields.Lookup( 1564 fname.substr(0, fname.length() - len)); 1565 if (field && field->value.type.base_type == basetype) 1566 return Error("Field " + fname + 1567 " would clash with generated functions for field " + 1568 field->name); 1569 } 1570 } 1571 return NoError(); 1572} 1573 1574static bool compareFieldDefs(const FieldDef *a, const FieldDef *b) { 1575 auto a_id = atoi(a->attributes.Lookup("id")->constant.c_str()); 1576 auto b_id = atoi(b->attributes.Lookup("id")->constant.c_str()); 1577 return a_id < b_id; 1578} 1579 1580CheckedError Parser::ParseDecl() { 1581 std::vector<std::string> dc = doc_comment_; 1582 bool fixed = Is(kTokenStruct); 1583 if (fixed) NEXT() else EXPECT(kTokenTable); 1584 std::string name = attribute_; 1585 EXPECT(kTokenIdentifier); 1586 StructDef *struct_def; 1587 ECHECK(StartStruct(name, &struct_def)); 1588 struct_def->doc_comment = dc; 1589 struct_def->fixed = fixed; 1590 ECHECK(ParseMetaData(&struct_def->attributes)); 1591 struct_def->sortbysize = 1592 struct_def->attributes.Lookup("original_order") == nullptr && !fixed; 1593 EXPECT('{'); 1594 while (token_ != '}') ECHECK(ParseField(*struct_def)); 1595 auto force_align = struct_def->attributes.Lookup("force_align"); 1596 if (fixed && force_align) { 1597 auto align = static_cast<size_t>(atoi(force_align->constant.c_str())); 1598 if (force_align->type.base_type != BASE_TYPE_INT || 1599 align < struct_def->minalign || 1600 align > FLATBUFFERS_MAX_ALIGNMENT || 1601 align & (align - 1)) 1602 return Error("force_align must be a power of two integer ranging from the" 1603 "struct\'s natural alignment to " + 1604 NumToString(FLATBUFFERS_MAX_ALIGNMENT)); 1605 struct_def->minalign = align; 1606 } 1607 struct_def->PadLastField(struct_def->minalign); 1608 // Check if this is a table that has manual id assignments 1609 auto &fields = struct_def->fields.vec; 1610 if (!struct_def->fixed && fields.size()) { 1611 size_t num_id_fields = 0; 1612 for (auto it = fields.begin(); it != fields.end(); ++it) { 1613 if ((*it)->attributes.Lookup("id")) num_id_fields++; 1614 } 1615 // If any fields have ids.. 1616 if (num_id_fields) { 1617 // Then all fields must have them. 1618 if (num_id_fields != fields.size()) 1619 return Error( 1620 "either all fields or no fields must have an 'id' attribute"); 1621 // Simply sort by id, then the fields are the same as if no ids had 1622 // been specified. 1623 std::sort(fields.begin(), fields.end(), compareFieldDefs); 1624 // Verify we have a contiguous set, and reassign vtable offsets. 1625 for (int i = 0; i < static_cast<int>(fields.size()); i++) { 1626 if (i != atoi(fields[i]->attributes.Lookup("id")->constant.c_str())) 1627 return Error("field id\'s must be consecutive from 0, id " + 1628 NumToString(i) + " missing or set twice"); 1629 fields[i]->value.offset = FieldIndexToOffset(static_cast<voffset_t>(i)); 1630 } 1631 } 1632 } 1633 1634 ECHECK(CheckClash(fields, struct_def, UnionTypeFieldSuffix(), 1635 BASE_TYPE_UNION)); 1636 ECHECK(CheckClash(fields, struct_def, "Type", BASE_TYPE_UNION)); 1637 ECHECK(CheckClash(fields, struct_def, "_length", BASE_TYPE_VECTOR)); 1638 ECHECK(CheckClash(fields, struct_def, "Length", BASE_TYPE_VECTOR)); 1639 ECHECK(CheckClash(fields, struct_def, "_byte_vector", BASE_TYPE_STRING)); 1640 ECHECK(CheckClash(fields, struct_def, "ByteVector", BASE_TYPE_STRING)); 1641 EXPECT('}'); 1642 types_.Add(namespaces_.back()->GetFullyQualifiedName(struct_def->name), 1643 new Type(BASE_TYPE_STRUCT, struct_def, nullptr)); 1644 return NoError(); 1645} 1646 1647CheckedError Parser::ParseService() { 1648 std::vector<std::string> service_comment = doc_comment_; 1649 NEXT(); 1650 auto service_name = attribute_; 1651 EXPECT(kTokenIdentifier); 1652 auto &service_def = *new ServiceDef(); 1653 service_def.name = service_name; 1654 service_def.file = file_being_parsed_; 1655 service_def.doc_comment = service_comment; 1656 service_def.defined_namespace = namespaces_.back(); 1657 if (services_.Add(namespaces_.back()->GetFullyQualifiedName(service_name), 1658 &service_def)) 1659 return Error("service already exists: " + service_name); 1660 ECHECK(ParseMetaData(&service_def.attributes)); 1661 EXPECT('{'); 1662 do { 1663 auto rpc_name = attribute_; 1664 EXPECT(kTokenIdentifier); 1665 EXPECT('('); 1666 Type reqtype, resptype; 1667 ECHECK(ParseTypeIdent(reqtype)); 1668 EXPECT(')'); 1669 EXPECT(':'); 1670 ECHECK(ParseTypeIdent(resptype)); 1671 if (reqtype.base_type != BASE_TYPE_STRUCT || reqtype.struct_def->fixed || 1672 resptype.base_type != BASE_TYPE_STRUCT || resptype.struct_def->fixed) 1673 return Error("rpc request and response types must be tables"); 1674 auto &rpc = *new RPCCall(); 1675 rpc.name = rpc_name; 1676 rpc.request = reqtype.struct_def; 1677 rpc.response = resptype.struct_def; 1678 if (service_def.calls.Add(rpc_name, &rpc)) 1679 return Error("rpc already exists: " + rpc_name); 1680 ECHECK(ParseMetaData(&rpc.attributes)); 1681 EXPECT(';'); 1682 } while (token_ != '}'); 1683 NEXT(); 1684 return NoError(); 1685} 1686 1687bool Parser::SetRootType(const char *name) { 1688 root_struct_def_ = structs_.Lookup(name); 1689 if (!root_struct_def_) 1690 root_struct_def_ = structs_.Lookup( 1691 namespaces_.back()->GetFullyQualifiedName(name)); 1692 return root_struct_def_ != nullptr; 1693} 1694 1695void Parser::MarkGenerated() { 1696 // This function marks all existing definitions as having already 1697 // been generated, which signals no code for included files should be 1698 // generated. 1699 for (auto it = enums_.vec.begin(); 1700 it != enums_.vec.end(); ++it) { 1701 (*it)->generated = true; 1702 } 1703 for (auto it = structs_.vec.begin(); 1704 it != structs_.vec.end(); ++it) { 1705 if (!(*it)->predecl) { 1706 (*it)->generated = true; 1707 } 1708 } 1709 for (auto it = services_.vec.begin(); 1710 it != services_.vec.end(); ++it) { 1711 (*it)->generated = true; 1712 } 1713} 1714 1715CheckedError Parser::ParseNamespace() { 1716 NEXT(); 1717 auto ns = new Namespace(); 1718 namespaces_.push_back(ns); 1719 if (token_ != ';') { 1720 for (;;) { 1721 ns->components.push_back(attribute_); 1722 EXPECT(kTokenIdentifier); 1723 if (Is('.')) NEXT() else break; 1724 } 1725 } 1726 EXPECT(';'); 1727 return NoError(); 1728} 1729 1730static bool compareEnumVals(const EnumVal *a, const EnumVal* b) { 1731 return a->value < b->value; 1732} 1733 1734// Best effort parsing of .proto declarations, with the aim to turn them 1735// in the closest corresponding FlatBuffer equivalent. 1736// We parse everything as identifiers instead of keywords, since we don't 1737// want protobuf keywords to become invalid identifiers in FlatBuffers. 1738CheckedError Parser::ParseProtoDecl() { 1739 bool isextend = attribute_ == "extend"; 1740 if (attribute_ == "package") { 1741 // These are identical in syntax to FlatBuffer's namespace decl. 1742 ECHECK(ParseNamespace()); 1743 } else if (attribute_ == "message" || isextend) { 1744 std::vector<std::string> struct_comment = doc_comment_; 1745 NEXT(); 1746 StructDef *struct_def = nullptr; 1747 if (isextend) { 1748 if (Is('.')) NEXT(); // qualified names may start with a . ? 1749 auto id = attribute_; 1750 EXPECT(kTokenIdentifier); 1751 ECHECK(ParseNamespacing(&id, nullptr)); 1752 struct_def = LookupCreateStruct(id, false); 1753 if (!struct_def) 1754 return Error("cannot extend unknown message type: " + id); 1755 } else { 1756 std::string name = attribute_; 1757 EXPECT(kTokenIdentifier); 1758 ECHECK(StartStruct(name, &struct_def)); 1759 // Since message definitions can be nested, we create a new namespace. 1760 auto ns = new Namespace(); 1761 // Copy of current namespace. 1762 *ns = *namespaces_.back(); 1763 // But with current message name. 1764 ns->components.push_back(name); 1765 namespaces_.push_back(ns); 1766 } 1767 struct_def->doc_comment = struct_comment; 1768 ECHECK(ParseProtoFields(struct_def, isextend, false)); 1769 if (!isextend) { 1770 // We have to remove the nested namespace, but we can't just throw it 1771 // away, so put it at the beginning of the vector. 1772 auto ns = namespaces_.back(); 1773 namespaces_.pop_back(); 1774 namespaces_.insert(namespaces_.begin(), ns); 1775 } 1776 if (Is(';')) NEXT(); 1777 } else if (attribute_ == "enum") { 1778 // These are almost the same, just with different terminator: 1779 EnumDef *enum_def; 1780 ECHECK(ParseEnum(false, &enum_def)); 1781 if (Is(';')) NEXT(); 1782 // Protobuf allows them to be specified in any order, so sort afterwards. 1783 auto &v = enum_def->vals.vec; 1784 std::sort(v.begin(), v.end(), compareEnumVals); 1785 1786 // Temp: remove any duplicates, as .fbs files can't handle them. 1787 for (auto it = v.begin(); it != v.end(); ) { 1788 if (it != v.begin() && it[0]->value == it[-1]->value) it = v.erase(it); 1789 else ++it; 1790 } 1791 } else if (attribute_ == "syntax") { // Skip these. 1792 NEXT(); 1793 EXPECT('='); 1794 EXPECT(kTokenStringConstant); 1795 EXPECT(';'); 1796 } else if (attribute_ == "option") { // Skip these. 1797 ECHECK(ParseProtoOption()); 1798 EXPECT(';'); 1799 } else if (attribute_ == "service") { // Skip these. 1800 NEXT(); 1801 EXPECT(kTokenIdentifier); 1802 ECHECK(ParseProtoCurliesOrIdent()); 1803 } else { 1804 return Error("don\'t know how to parse .proto declaration starting with " + 1805 TokenToStringId(token_)); 1806 } 1807 return NoError(); 1808} 1809 1810CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend, 1811 bool inside_oneof) { 1812 EXPECT('{'); 1813 while (token_ != '}') { 1814 if (attribute_ == "message" || attribute_ == "extend" || 1815 attribute_ == "enum") { 1816 // Nested declarations. 1817 ECHECK(ParseProtoDecl()); 1818 } else if (attribute_ == "extensions") { // Skip these. 1819 NEXT(); 1820 EXPECT(kTokenIntegerConstant); 1821 if (Is(kTokenIdentifier)) { 1822 NEXT(); // to 1823 NEXT(); // num 1824 } 1825 EXPECT(';'); 1826 } else if (attribute_ == "option") { // Skip these. 1827 ECHECK(ParseProtoOption()); 1828 EXPECT(';'); 1829 } else if (attribute_ == "reserved") { // Skip these. 1830 NEXT(); 1831 EXPECT(kTokenIntegerConstant); 1832 while (Is(',')) { NEXT(); EXPECT(kTokenIntegerConstant); } 1833 EXPECT(';'); 1834 } else { 1835 std::vector<std::string> field_comment = doc_comment_; 1836 // Parse the qualifier. 1837 bool required = false; 1838 bool repeated = false; 1839 bool oneof = false; 1840 if (!inside_oneof) { 1841 if (attribute_ == "optional") { 1842 // This is the default. 1843 EXPECT(kTokenIdentifier); 1844 } else if (attribute_ == "required") { 1845 required = true; 1846 EXPECT(kTokenIdentifier); 1847 } else if (attribute_ == "repeated") { 1848 repeated = true; 1849 EXPECT(kTokenIdentifier); 1850 } else if (attribute_ == "oneof") { 1851 oneof = true; 1852 EXPECT(kTokenIdentifier); 1853 } else { 1854 // can't error, proto3 allows decls without any of the above. 1855 } 1856 } 1857 StructDef *anonymous_struct = nullptr; 1858 Type type; 1859 if (attribute_ == "group" || oneof) { 1860 if (!oneof) EXPECT(kTokenIdentifier); 1861 auto name = "Anonymous" + NumToString(anonymous_counter++); 1862 ECHECK(StartStruct(name, &anonymous_struct)); 1863 type = Type(BASE_TYPE_STRUCT, anonymous_struct); 1864 } else { 1865 ECHECK(ParseTypeFromProtoType(&type)); 1866 } 1867 // Repeated elements get mapped to a vector. 1868 if (repeated) { 1869 type.element = type.base_type; 1870 type.base_type = BASE_TYPE_VECTOR; 1871 } 1872 std::string name = attribute_; 1873 // Protos may use our keywords "attribute" & "namespace" as an identifier. 1874 if (Is(kTokenAttribute) || Is(kTokenNameSpace)) { 1875 NEXT(); 1876 // TODO: simpler to just not make these keywords? 1877 name += "_"; // Have to make it not a keyword. 1878 } else { 1879 EXPECT(kTokenIdentifier); 1880 } 1881 if (!oneof) { 1882 // Parse the field id. Since we're just translating schemas, not 1883 // any kind of binary compatibility, we can safely ignore these, and 1884 // assign our own. 1885 EXPECT('='); 1886 EXPECT(kTokenIntegerConstant); 1887 } 1888 FieldDef *field = nullptr; 1889 if (isextend) { 1890 // We allow a field to be re-defined when extending. 1891 // TODO: are there situations where that is problematic? 1892 field = struct_def->fields.Lookup(name); 1893 } 1894 if (!field) ECHECK(AddField(*struct_def, name, type, &field)); 1895 field->doc_comment = field_comment; 1896 if (!IsScalar(type.base_type)) field->required = required; 1897 // See if there's a default specified. 1898 if (Is('[')) { 1899 NEXT(); 1900 for (;;) { 1901 auto key = attribute_; 1902 ECHECK(ParseProtoKey()); 1903 EXPECT('='); 1904 auto val = attribute_; 1905 ECHECK(ParseProtoCurliesOrIdent()); 1906 if (key == "default") { 1907 // Temp: skip non-numeric defaults (enums). 1908 auto numeric = strpbrk(val.c_str(), "0123456789-+."); 1909 if (IsScalar(type.base_type) && numeric == val.c_str()) 1910 field->value.constant = val; 1911 } else if (key == "deprecated") { 1912 field->deprecated = val == "true"; 1913 } 1914 if (!Is(',')) break; 1915 NEXT(); 1916 } 1917 EXPECT(']'); 1918 } 1919 if (anonymous_struct) { 1920 ECHECK(ParseProtoFields(anonymous_struct, false, oneof)); 1921 if (Is(';')) NEXT(); 1922 } else { 1923 EXPECT(';'); 1924 } 1925 } 1926 } 1927 NEXT(); 1928 return NoError(); 1929} 1930 1931CheckedError Parser::ParseProtoKey() { 1932 if (token_ == '(') { 1933 NEXT(); 1934 // Skip "(a.b)" style custom attributes. 1935 while (token_ == '.' || token_ == kTokenIdentifier) NEXT(); 1936 EXPECT(')'); 1937 while (Is('.')) { NEXT(); EXPECT(kTokenIdentifier); } 1938 } else { 1939 EXPECT(kTokenIdentifier); 1940 } 1941 return NoError(); 1942} 1943 1944CheckedError Parser::ParseProtoCurliesOrIdent() { 1945 if (Is('{')) { 1946 NEXT(); 1947 for (int nesting = 1; nesting; ) { 1948 if (token_ == '{') nesting++; 1949 else if (token_ == '}') nesting--; 1950 NEXT(); 1951 } 1952 } else { 1953 NEXT(); // Any single token. 1954 } 1955 return NoError(); 1956} 1957 1958CheckedError Parser::ParseProtoOption() { 1959 NEXT(); 1960 ECHECK(ParseProtoKey()); 1961 EXPECT('='); 1962 ECHECK(ParseProtoCurliesOrIdent()); 1963 return NoError(); 1964} 1965 1966// Parse a protobuf type, and map it to the corresponding FlatBuffer one. 1967CheckedError Parser::ParseTypeFromProtoType(Type *type) { 1968 struct type_lookup { const char *proto_type; BaseType fb_type; }; 1969 static type_lookup lookup[] = { 1970 { "float", BASE_TYPE_FLOAT }, { "double", BASE_TYPE_DOUBLE }, 1971 { "int32", BASE_TYPE_INT }, { "int64", BASE_TYPE_LONG }, 1972 { "uint32", BASE_TYPE_UINT }, { "uint64", BASE_TYPE_ULONG }, 1973 { "sint32", BASE_TYPE_INT }, { "sint64", BASE_TYPE_LONG }, 1974 { "fixed32", BASE_TYPE_UINT }, { "fixed64", BASE_TYPE_ULONG }, 1975 { "sfixed32", BASE_TYPE_INT }, { "sfixed64", BASE_TYPE_LONG }, 1976 { "bool", BASE_TYPE_BOOL }, 1977 { "string", BASE_TYPE_STRING }, 1978 { "bytes", BASE_TYPE_STRING }, 1979 { nullptr, BASE_TYPE_NONE } 1980 }; 1981 for (auto tl = lookup; tl->proto_type; tl++) { 1982 if (attribute_ == tl->proto_type) { 1983 type->base_type = tl->fb_type; 1984 NEXT(); 1985 return NoError(); 1986 } 1987 } 1988 if (Is('.')) NEXT(); // qualified names may start with a . ? 1989 ECHECK(ParseTypeIdent(*type)); 1990 return NoError(); 1991} 1992 1993CheckedError Parser::SkipAnyJsonValue() { 1994 switch (token_) { 1995 case '{': { 1996 size_t fieldn_outer = 0; 1997 return ParseTableDelimiters(fieldn_outer, nullptr, 1998 [](const std::string &, 1999 size_t &fieldn, const StructDef *, 2000 void *state) -> CheckedError { 2001 auto *parser = static_cast<Parser *>(state); 2002 ECHECK(parser->SkipAnyJsonValue()); 2003 fieldn++; 2004 return NoError(); 2005 }, 2006 this); 2007 } 2008 case '[': { 2009 size_t count = 0; 2010 return ParseVectorDelimiters(count, [](size_t &, 2011 void *state) -> CheckedError { 2012 return static_cast<Parser *>(state)->SkipAnyJsonValue(); 2013 }, 2014 this); 2015 } 2016 case kTokenStringConstant: 2017 EXPECT(kTokenStringConstant); 2018 break; 2019 case kTokenIntegerConstant: 2020 EXPECT(kTokenIntegerConstant); 2021 break; 2022 case kTokenFloatConstant: 2023 EXPECT(kTokenFloatConstant); 2024 break; 2025 case kTokenBooleanConstant: 2026 EXPECT(kTokenBooleanConstant); 2027 break; 2028 default: 2029 return TokenError(); 2030 } 2031 return NoError(); 2032} 2033 2034CheckedError Parser::ParseFlexBufferValue(flexbuffers::Builder *builder) { 2035 switch (token_) { 2036 case '{': { 2037 std::pair<Parser *, flexbuffers::Builder *> parser_and_builder_state( 2038 this, builder); 2039 auto start = builder->StartMap(); 2040 size_t fieldn_outer = 0; 2041 auto err = ParseTableDelimiters(fieldn_outer, nullptr, 2042 [](const std::string &name, 2043 size_t &fieldn, const StructDef *, 2044 void *state) -> CheckedError { 2045 auto *parser_and_builder = 2046 static_cast<std::pair<Parser *, flexbuffers::Builder *> *>( 2047 state); 2048 auto *parser = parser_and_builder->first; 2049 auto *current_builder = parser_and_builder->second; 2050 current_builder->Key(name); 2051 ECHECK(parser->ParseFlexBufferValue(current_builder)); 2052 fieldn++; 2053 return NoError(); 2054 }, 2055 &parser_and_builder_state); 2056 ECHECK(err); 2057 builder->EndMap(start); 2058 break; 2059 } 2060 case '[':{ 2061 auto start = builder->StartVector(); 2062 size_t count = 0; 2063 std::pair<Parser *, flexbuffers::Builder *> parser_and_builder_state( 2064 this, builder); 2065 ECHECK(ParseVectorDelimiters(count, [](size_t &, 2066 void *state) -> CheckedError { 2067 auto *parser_and_builder = 2068 static_cast<std::pair<Parser *, flexbuffers::Builder *> *>( 2069 state); 2070 return parser_and_builder->first->ParseFlexBufferValue( 2071 parser_and_builder->second); 2072 }, 2073 &parser_and_builder_state)); 2074 builder->EndVector(start, false, false); 2075 break; 2076 } 2077 case kTokenStringConstant: 2078 builder->String(attribute_); 2079 EXPECT(kTokenStringConstant); 2080 break; 2081 case kTokenIntegerConstant: 2082 builder->Int(StringToInt(attribute_.c_str())); 2083 EXPECT(kTokenIntegerConstant); 2084 break; 2085 case kTokenBooleanConstant: 2086 builder->Bool(StringToInt(attribute_.c_str()) != 0); 2087 EXPECT(kTokenBooleanConstant); 2088 break; 2089 case kTokenFloatConstant: 2090 builder->Double(strtod(attribute_.c_str(), nullptr)); 2091 EXPECT(kTokenFloatConstant); 2092 break; 2093 default: 2094 return TokenError(); 2095 } 2096 return NoError(); 2097} 2098 2099bool Parser::ParseFlexBuffer(const char *source, const char *source_filename, 2100 flexbuffers::Builder *builder) { 2101 auto ok = !StartParseFile(source, source_filename).Check() && 2102 !ParseFlexBufferValue(builder).Check(); 2103 if (ok) builder->Finish(); 2104 return ok; 2105} 2106 2107bool Parser::Parse(const char *source, const char **include_paths, 2108 const char *source_filename) { 2109 return !ParseRoot(source, include_paths, source_filename).Check(); 2110} 2111 2112CheckedError Parser::StartParseFile(const char *source, const char *source_filename) { 2113 file_being_parsed_ = source_filename ? source_filename : ""; 2114 source_ = cursor_ = source; 2115 line_ = 1; 2116 error_.clear(); 2117 ECHECK(SkipByteOrderMark()); 2118 NEXT(); 2119 if (Is(kTokenEof)) 2120 return Error("input file is empty"); 2121 return NoError(); 2122} 2123 2124CheckedError Parser::ParseRoot(const char *source, const char **include_paths, 2125 const char *source_filename) { 2126 ECHECK(DoParse(source, include_paths, source_filename, nullptr)); 2127 2128 // Check that all types were defined. 2129 for (auto it = structs_.vec.begin(); it != structs_.vec.end(); ++it) { 2130 if ((*it)->predecl) { 2131 return Error("type referenced but not defined: " + (*it)->name); 2132 } 2133 } 2134 2135 // This check has to happen here and not earlier, because only now do we 2136 // know for sure what the type of these are. 2137 for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) { 2138 auto &enum_def = **it; 2139 if (enum_def.is_union) { 2140 for (auto val_it = enum_def.vals.vec.begin(); 2141 val_it != enum_def.vals.vec.end(); 2142 ++val_it) { 2143 auto &val = **val_it; 2144 if (opts.lang_to_generate != IDLOptions::kCpp && 2145 val.union_type.struct_def && val.union_type.struct_def->fixed) 2146 return Error( 2147 "only tables can be union elements in the generated language: " 2148 + val.name); 2149 } 2150 } 2151 } 2152 return NoError(); 2153} 2154 2155CheckedError Parser::DoParse(const char *source, 2156 const char **include_paths, 2157 const char *source_filename, 2158 const char *include_filename) { 2159 if (source_filename && 2160 included_files_.find(source_filename) == included_files_.end()) { 2161 included_files_[source_filename] = include_filename ? include_filename : ""; 2162 files_included_per_file_[source_filename] = std::set<std::string>(); 2163 } 2164 if (!include_paths) { 2165 static const char *current_directory[] = { "", nullptr }; 2166 include_paths = current_directory; 2167 } 2168 field_stack_.clear(); 2169 builder_.Clear(); 2170 // Start with a blank namespace just in case this file doesn't have one. 2171 namespaces_.push_back(new Namespace()); 2172 2173 ECHECK(StartParseFile(source, source_filename)); 2174 2175 // Includes must come before type declarations: 2176 for (;;) { 2177 // Parse pre-include proto statements if any: 2178 if (opts.proto_mode && 2179 (attribute_ == "option" || attribute_ == "syntax" || 2180 attribute_ == "package")) { 2181 ECHECK(ParseProtoDecl()); 2182 } else if (Is(kTokenNativeInclude)) { 2183 NEXT(); 2184 vector_emplace_back(&native_included_files_, attribute_); 2185 EXPECT(kTokenStringConstant); 2186 } else if (Is(kTokenInclude) || 2187 (opts.proto_mode && 2188 attribute_ == "import" && 2189 Is(kTokenIdentifier))) { 2190 NEXT(); 2191 if (opts.proto_mode && attribute_ == "public") NEXT(); 2192 auto name = flatbuffers::PosixPath(attribute_.c_str()); 2193 EXPECT(kTokenStringConstant); 2194 // Look for the file in include_paths. 2195 std::string filepath; 2196 for (auto paths = include_paths; paths && *paths; paths++) { 2197 filepath = flatbuffers::ConCatPathFileName(*paths, name); 2198 if(FileExists(filepath.c_str())) break; 2199 } 2200 if (filepath.empty()) 2201 return Error("unable to locate include file: " + name); 2202 if (source_filename) 2203 files_included_per_file_[source_filename].insert(filepath); 2204 if (included_files_.find(filepath) == included_files_.end()) { 2205 // We found an include file that we have not parsed yet. 2206 // Load it and parse it. 2207 std::string contents; 2208 if (!LoadFile(filepath.c_str(), true, &contents)) 2209 return Error("unable to load include file: " + name); 2210 ECHECK(DoParse(contents.c_str(), include_paths, filepath.c_str(), 2211 name.c_str())); 2212 // We generally do not want to output code for any included files: 2213 if (!opts.generate_all) MarkGenerated(); 2214 // This is the easiest way to continue this file after an include: 2215 // instead of saving and restoring all the state, we simply start the 2216 // file anew. This will cause it to encounter the same include 2217 // statement again, but this time it will skip it, because it was 2218 // entered into included_files_. 2219 // This is recursive, but only go as deep as the number of include 2220 // statements. 2221 return DoParse(source, include_paths, source_filename, include_filename); 2222 } 2223 EXPECT(';'); 2224 } else { 2225 break; 2226 } 2227 } 2228 // Now parse all other kinds of declarations: 2229 while (token_ != kTokenEof) { 2230 if (opts.proto_mode) { 2231 ECHECK(ParseProtoDecl()); 2232 } else if (token_ == kTokenNameSpace) { 2233 ECHECK(ParseNamespace()); 2234 } else if (token_ == '{') { 2235 if (!root_struct_def_) 2236 return Error("no root type set to parse json with"); 2237 if (builder_.GetSize()) { 2238 return Error("cannot have more than one json object in a file"); 2239 } 2240 uoffset_t toff; 2241 ECHECK(ParseTable(*root_struct_def_, nullptr, &toff)); 2242 builder_.Finish(Offset<Table>(toff), 2243 file_identifier_.length() ? file_identifier_.c_str() : nullptr); 2244 } else if (token_ == kTokenEnum) { 2245 ECHECK(ParseEnum(false, nullptr)); 2246 } else if (token_ == kTokenUnion) { 2247 ECHECK(ParseEnum(true, nullptr)); 2248 } else if (token_ == kTokenRootType) { 2249 NEXT(); 2250 auto root_type = attribute_; 2251 EXPECT(kTokenIdentifier); 2252 ECHECK(ParseNamespacing(&root_type, nullptr)); 2253 if (!SetRootType(root_type.c_str())) 2254 return Error("unknown root type: " + root_type); 2255 if (root_struct_def_->fixed) 2256 return Error("root type must be a table"); 2257 EXPECT(';'); 2258 } else if (token_ == kTokenFileIdentifier) { 2259 NEXT(); 2260 file_identifier_ = attribute_; 2261 EXPECT(kTokenStringConstant); 2262 if (file_identifier_.length() != 2263 FlatBufferBuilder::kFileIdentifierLength) 2264 return Error("file_identifier must be exactly " + 2265 NumToString(FlatBufferBuilder::kFileIdentifierLength) + 2266 " characters"); 2267 EXPECT(';'); 2268 } else if (token_ == kTokenFileExtension) { 2269 NEXT(); 2270 file_extension_ = attribute_; 2271 EXPECT(kTokenStringConstant); 2272 EXPECT(';'); 2273 } else if(token_ == kTokenInclude) { 2274 return Error("includes must come before declarations"); 2275 } else if(token_ == kTokenAttribute) { 2276 NEXT(); 2277 auto name = attribute_; 2278 EXPECT(kTokenStringConstant); 2279 EXPECT(';'); 2280 known_attributes_[name] = false; 2281 } else if (token_ == kTokenService) { 2282 ECHECK(ParseService()); 2283 } else { 2284 ECHECK(ParseDecl()); 2285 } 2286 } 2287 return NoError(); 2288} 2289 2290std::set<std::string> Parser::GetIncludedFilesRecursive( 2291 const std::string &file_name) const { 2292 std::set<std::string> included_files; 2293 std::list<std::string> to_process; 2294 2295 if (file_name.empty()) return included_files; 2296 to_process.push_back(file_name); 2297 2298 while (!to_process.empty()) { 2299 std::string current = to_process.front(); 2300 to_process.pop_front(); 2301 included_files.insert(current); 2302 2303 // Workaround the lack of const accessor in C++98 maps. 2304 auto &new_files = 2305 (*const_cast<std::map<std::string, std::set<std::string>> *>( 2306 &files_included_per_file_))[current]; 2307 for (auto it = new_files.begin(); it != new_files.end(); ++it) { 2308 if (included_files.find(*it) == included_files.end()) 2309 to_process.push_back(*it); 2310 } 2311 } 2312 2313 return included_files; 2314} 2315 2316// Schema serialization functionality: 2317 2318template<typename T> bool compareName(const T* a, const T* b) { 2319 return a->defined_namespace->GetFullyQualifiedName(a->name) 2320 < b->defined_namespace->GetFullyQualifiedName(b->name); 2321} 2322 2323template<typename T> void AssignIndices(const std::vector<T *> &defvec) { 2324 // Pre-sort these vectors, such that we can set the correct indices for them. 2325 auto vec = defvec; 2326 std::sort(vec.begin(), vec.end(), compareName<T>); 2327 for (int i = 0; i < static_cast<int>(vec.size()); i++) vec[i]->index = i; 2328} 2329 2330void Parser::Serialize() { 2331 builder_.Clear(); 2332 AssignIndices(structs_.vec); 2333 AssignIndices(enums_.vec); 2334 std::vector<Offset<reflection::Object>> object_offsets; 2335 for (auto it = structs_.vec.begin(); it != structs_.vec.end(); ++it) { 2336 auto offset = (*it)->Serialize(&builder_, *this); 2337 object_offsets.push_back(offset); 2338 (*it)->serialized_location = offset.o; 2339 } 2340 std::vector<Offset<reflection::Enum>> enum_offsets; 2341 for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) { 2342 auto offset = (*it)->Serialize(&builder_, *this); 2343 enum_offsets.push_back(offset); 2344 (*it)->serialized_location = offset.o; 2345 } 2346 auto schema_offset = reflection::CreateSchema( 2347 builder_, 2348 builder_.CreateVectorOfSortedTables(&object_offsets), 2349 builder_.CreateVectorOfSortedTables(&enum_offsets), 2350 builder_.CreateString(file_identifier_), 2351 builder_.CreateString(file_extension_), 2352 root_struct_def_ 2353 ? root_struct_def_->serialized_location 2354 : 0); 2355 builder_.Finish(schema_offset, reflection::SchemaIdentifier()); 2356} 2357 2358Offset<reflection::Object> StructDef::Serialize(FlatBufferBuilder *builder, 2359 const Parser &parser) const { 2360 std::vector<Offset<reflection::Field>> field_offsets; 2361 for (auto it = fields.vec.begin(); it != fields.vec.end(); ++it) { 2362 field_offsets.push_back( 2363 (*it)->Serialize(builder, 2364 static_cast<uint16_t>(it - fields.vec.begin()), parser)); 2365 } 2366 auto qualified_name = defined_namespace->GetFullyQualifiedName(name); 2367 return reflection::CreateObject(*builder, 2368 builder->CreateString(qualified_name), 2369 builder->CreateVectorOfSortedTables( 2370 &field_offsets), 2371 fixed, 2372 static_cast<int>(minalign), 2373 static_cast<int>(bytesize), 2374 SerializeAttributes(builder, parser), 2375 parser.opts.binary_schema_comments 2376 ? builder->CreateVectorOfStrings( 2377 doc_comment) 2378 : 0); 2379} 2380 2381Offset<reflection::Field> FieldDef::Serialize(FlatBufferBuilder *builder, 2382 uint16_t id, 2383 const Parser &parser) const { 2384 return reflection::CreateField(*builder, 2385 builder->CreateString(name), 2386 value.type.Serialize(builder), 2387 id, 2388 value.offset, 2389 IsInteger(value.type.base_type) 2390 ? StringToInt(value.constant.c_str()) 2391 : 0, 2392 IsFloat(value.type.base_type) 2393 ? strtod(value.constant.c_str(), nullptr) 2394 : 0.0, 2395 deprecated, 2396 required, 2397 key, 2398 SerializeAttributes(builder, parser), 2399 parser.opts.binary_schema_comments 2400 ? builder->CreateVectorOfStrings(doc_comment) 2401 : 0); 2402 // TODO: value.constant is almost always "0", we could save quite a bit of 2403 // space by sharing it. Same for common values of value.type. 2404} 2405 2406Offset<reflection::Enum> EnumDef::Serialize(FlatBufferBuilder *builder, 2407 const Parser &parser) const { 2408 std::vector<Offset<reflection::EnumVal>> enumval_offsets; 2409 for (auto it = vals.vec.begin(); it != vals.vec.end(); ++it) { 2410 enumval_offsets.push_back((*it)->Serialize(builder)); 2411 } 2412 auto qualified_name = defined_namespace->GetFullyQualifiedName(name); 2413 return reflection::CreateEnum(*builder, 2414 builder->CreateString(qualified_name), 2415 builder->CreateVector(enumval_offsets), 2416 is_union, 2417 underlying_type.Serialize(builder), 2418 SerializeAttributes(builder, parser), 2419 parser.opts.binary_schema_comments 2420 ? builder->CreateVectorOfStrings(doc_comment) 2421 : 0); 2422} 2423 2424Offset<reflection::EnumVal> EnumVal::Serialize(FlatBufferBuilder *builder) const 2425 { 2426 return reflection::CreateEnumVal(*builder, 2427 builder->CreateString(name), 2428 value, 2429 union_type.struct_def 2430 ? union_type.struct_def-> 2431 serialized_location 2432 : 0, 2433 union_type.Serialize(builder)); 2434} 2435 2436Offset<reflection::Type> Type::Serialize(FlatBufferBuilder *builder) const { 2437 return reflection::CreateType(*builder, 2438 static_cast<reflection::BaseType>(base_type), 2439 static_cast<reflection::BaseType>(element), 2440 struct_def ? struct_def->index : 2441 (enum_def ? enum_def->index : -1)); 2442} 2443 2444flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset< 2445 reflection::KeyValue>>> 2446 Definition::SerializeAttributes(FlatBufferBuilder *builder, 2447 const Parser &parser) const { 2448 std::vector<flatbuffers::Offset<reflection::KeyValue>> attrs; 2449 for (auto kv = attributes.dict.begin(); kv != attributes.dict.end(); ++kv) { 2450 auto it = parser.known_attributes_.find(kv->first); 2451 assert(it != parser.known_attributes_.end()); 2452 if (!it->second) { // Custom attribute. 2453 attrs.push_back( 2454 reflection::CreateKeyValue(*builder, builder->CreateString(kv->first), 2455 builder->CreateString( 2456 kv->second->constant))); 2457 } 2458 } 2459 if (attrs.size()) { 2460 return builder->CreateVectorOfSortedTables(&attrs); 2461 } else { 2462 return 0; 2463 } 2464} 2465 2466std::string Parser::ConformTo(const Parser &base) { 2467 for (auto sit = structs_.vec.begin(); sit != structs_.vec.end(); ++sit) { 2468 auto &struct_def = **sit; 2469 auto qualified_name = 2470 struct_def.defined_namespace->GetFullyQualifiedName(struct_def.name); 2471 auto struct_def_base = base.structs_.Lookup(qualified_name); 2472 if (!struct_def_base) continue; 2473 for (auto fit = struct_def.fields.vec.begin(); 2474 fit != struct_def.fields.vec.end(); ++fit) { 2475 auto &field = **fit; 2476 auto field_base = struct_def_base->fields.Lookup(field.name); 2477 if (field_base) { 2478 if (field.value.offset != field_base->value.offset) 2479 return "offsets differ for field: " + field.name; 2480 if (field.value.constant != field_base->value.constant) 2481 return "defaults differ for field: " + field.name; 2482 if (!EqualByName(field.value.type, field_base->value.type)) 2483 return "types differ for field: " + field.name; 2484 } else { 2485 // Doesn't have to exist, deleting fields is fine. 2486 // But we should check if there is a field that has the same offset 2487 // but is incompatible (in the case of field renaming). 2488 for (auto fbit = struct_def_base->fields.vec.begin(); 2489 fbit != struct_def_base->fields.vec.end(); ++fbit) { 2490 field_base = *fbit; 2491 if (field.value.offset == field_base->value.offset) { 2492 if (!EqualByName(field.value.type, field_base->value.type)) 2493 return "field renamed to different type: " + field.name; 2494 break; 2495 } 2496 } 2497 } 2498 } 2499 } 2500 for (auto eit = enums_.vec.begin(); eit != enums_.vec.end(); ++eit) { 2501 auto &enum_def = **eit; 2502 auto qualified_name = 2503 enum_def.defined_namespace->GetFullyQualifiedName(enum_def.name); 2504 auto enum_def_base = base.enums_.Lookup(qualified_name); 2505 if (!enum_def_base) continue; 2506 for (auto evit = enum_def.vals.vec.begin(); 2507 evit != enum_def.vals.vec.end(); ++evit) { 2508 auto &enum_val = **evit; 2509 auto enum_val_base = enum_def_base->vals.Lookup(enum_val.name); 2510 if (enum_val_base) { 2511 if (enum_val.value != enum_val_base->value) 2512 return "values differ for enum: " + enum_val.name; 2513 } 2514 } 2515 } 2516 return ""; 2517} 2518 2519} // namespace flatbuffers 2520