1// Protocol Buffers - Google's data interchange format
2// Copyright 2008 Google Inc.  All rights reserved.
3// http://code.google.com/p/protobuf/
4//
5// Redistribution and use in source and binary forms, with or without
6// modification, are permitted provided that the following conditions are
7// met:
8//
9//     * Redistributions of source code must retain the above copyright
10// notice, this list of conditions and the following disclaimer.
11//     * Redistributions in binary form must reproduce the above
12// copyright notice, this list of conditions and the following disclaimer
13// in the documentation and/or other materials provided with the
14// distribution.
15//     * Neither the name of Google Inc. nor the names of its
16// contributors may be used to endorse or promote products derived from
17// this software without specific prior written permission.
18//
19// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31// Author: kenton@google.com (Kenton Varda)
32//  Based on original Protocol Buffers design by
33//  Sanjay Ghemawat, Jeff Dean, and others.
34//
35// Recursive descent FTW.
36
37#include <float.h>
38#include <google/protobuf/stubs/hash.h>
39#include <limits>
40
41
42#include <google/protobuf/compiler/parser.h>
43#include <google/protobuf/descriptor.h>
44#include <google/protobuf/descriptor.pb.h>
45#include <google/protobuf/wire_format.h>
46#include <google/protobuf/io/tokenizer.h>
47#include <google/protobuf/stubs/common.h>
48#include <google/protobuf/stubs/strutil.h>
49#include <google/protobuf/stubs/map-util.h>
50
51namespace google {
52namespace protobuf {
53namespace compiler {
54
55using internal::WireFormat;
56
57namespace {
58
59typedef hash_map<string, FieldDescriptorProto::Type> TypeNameMap;
60
61TypeNameMap MakeTypeNameTable() {
62  TypeNameMap result;
63
64  result["double"  ] = FieldDescriptorProto::TYPE_DOUBLE;
65  result["float"   ] = FieldDescriptorProto::TYPE_FLOAT;
66  result["uint64"  ] = FieldDescriptorProto::TYPE_UINT64;
67  result["fixed64" ] = FieldDescriptorProto::TYPE_FIXED64;
68  result["fixed32" ] = FieldDescriptorProto::TYPE_FIXED32;
69  result["bool"    ] = FieldDescriptorProto::TYPE_BOOL;
70  result["string"  ] = FieldDescriptorProto::TYPE_STRING;
71  result["group"   ] = FieldDescriptorProto::TYPE_GROUP;
72
73  result["bytes"   ] = FieldDescriptorProto::TYPE_BYTES;
74  result["uint32"  ] = FieldDescriptorProto::TYPE_UINT32;
75  result["sfixed32"] = FieldDescriptorProto::TYPE_SFIXED32;
76  result["sfixed64"] = FieldDescriptorProto::TYPE_SFIXED64;
77  result["int32"   ] = FieldDescriptorProto::TYPE_INT32;
78  result["int64"   ] = FieldDescriptorProto::TYPE_INT64;
79  result["sint32"  ] = FieldDescriptorProto::TYPE_SINT32;
80  result["sint64"  ] = FieldDescriptorProto::TYPE_SINT64;
81
82  return result;
83}
84
85const TypeNameMap kTypeNames = MakeTypeNameTable();
86
87}  // anonymous namespace
88
89// Makes code slightly more readable.  The meaning of "DO(foo)" is
90// "Execute foo and fail if it fails.", where failure is indicated by
91// returning false.
92#define DO(STATEMENT) if (STATEMENT) {} else return false
93
94// ===================================================================
95
96Parser::Parser()
97  : input_(NULL),
98    error_collector_(NULL),
99    source_location_table_(NULL),
100    had_errors_(false),
101    require_syntax_identifier_(false),
102    stop_after_syntax_identifier_(false) {
103}
104
105Parser::~Parser() {
106}
107
108// ===================================================================
109
110inline bool Parser::LookingAt(const char* text) {
111  return input_->current().text == text;
112}
113
114inline bool Parser::LookingAtType(io::Tokenizer::TokenType token_type) {
115  return input_->current().type == token_type;
116}
117
118inline bool Parser::AtEnd() {
119  return LookingAtType(io::Tokenizer::TYPE_END);
120}
121
122bool Parser::TryConsume(const char* text) {
123  if (LookingAt(text)) {
124    input_->Next();
125    return true;
126  } else {
127    return false;
128  }
129}
130
131bool Parser::Consume(const char* text, const char* error) {
132  if (TryConsume(text)) {
133    return true;
134  } else {
135    AddError(error);
136    return false;
137  }
138}
139
140bool Parser::Consume(const char* text) {
141  if (TryConsume(text)) {
142    return true;
143  } else {
144    AddError("Expected \"" + string(text) + "\".");
145    return false;
146  }
147}
148
149bool Parser::ConsumeIdentifier(string* output, const char* error) {
150  if (LookingAtType(io::Tokenizer::TYPE_IDENTIFIER)) {
151    *output = input_->current().text;
152    input_->Next();
153    return true;
154  } else {
155    AddError(error);
156    return false;
157  }
158}
159
160bool Parser::ConsumeInteger(int* output, const char* error) {
161  if (LookingAtType(io::Tokenizer::TYPE_INTEGER)) {
162    uint64 value = 0;
163    if (!io::Tokenizer::ParseInteger(input_->current().text,
164                                     kint32max, &value)) {
165      AddError("Integer out of range.");
166      // We still return true because we did, in fact, parse an integer.
167    }
168    *output = value;
169    input_->Next();
170    return true;
171  } else {
172    AddError(error);
173    return false;
174  }
175}
176
177bool Parser::ConsumeSignedInteger(int* output, const char* error) {
178  bool is_negative = false;
179  uint64 max_value = kint32max;
180  if (TryConsume("-")) {
181    is_negative = true;
182    max_value += 1;
183  }
184  uint64 value = 0;
185  DO(ConsumeInteger64(max_value, &value, error));
186  if (is_negative) value *= -1;
187  *output = value;
188  return true;
189}
190
191bool Parser::ConsumeInteger64(uint64 max_value, uint64* output,
192                              const char* error) {
193  if (LookingAtType(io::Tokenizer::TYPE_INTEGER)) {
194    if (!io::Tokenizer::ParseInteger(input_->current().text, max_value,
195                                     output)) {
196      AddError("Integer out of range.");
197      // We still return true because we did, in fact, parse an integer.
198      *output = 0;
199    }
200    input_->Next();
201    return true;
202  } else {
203    AddError(error);
204    return false;
205  }
206}
207
208bool Parser::ConsumeNumber(double* output, const char* error) {
209  if (LookingAtType(io::Tokenizer::TYPE_FLOAT)) {
210    *output = io::Tokenizer::ParseFloat(input_->current().text);
211    input_->Next();
212    return true;
213  } else if (LookingAtType(io::Tokenizer::TYPE_INTEGER)) {
214    // Also accept integers.
215    uint64 value = 0;
216    if (!io::Tokenizer::ParseInteger(input_->current().text,
217                                     kuint64max, &value)) {
218      AddError("Integer out of range.");
219      // We still return true because we did, in fact, parse a number.
220    }
221    *output = value;
222    input_->Next();
223    return true;
224  } else if (LookingAt("inf")) {
225    *output = numeric_limits<double>::infinity();
226    input_->Next();
227    return true;
228  } else if (LookingAt("nan")) {
229    *output = numeric_limits<double>::quiet_NaN();
230    input_->Next();
231    return true;
232  } else {
233    AddError(error);
234    return false;
235  }
236}
237
238bool Parser::ConsumeString(string* output, const char* error) {
239  if (LookingAtType(io::Tokenizer::TYPE_STRING)) {
240    io::Tokenizer::ParseString(input_->current().text, output);
241    input_->Next();
242    // Allow C++ like concatenation of adjacent string tokens.
243    while (LookingAtType(io::Tokenizer::TYPE_STRING)) {
244      io::Tokenizer::ParseStringAppend(input_->current().text, output);
245      input_->Next();
246    }
247    return true;
248  } else {
249    AddError(error);
250    return false;
251  }
252}
253
254bool Parser::TryConsumeEndOfDeclaration(const char* text,
255                                        const LocationRecorder* location) {
256  if (LookingAt(text)) {
257    string leading, trailing;
258    input_->NextWithComments(&trailing, NULL, &leading);
259
260    // Save the leading comments for next time, and recall the leading comments
261    // from last time.
262    leading.swap(upcoming_doc_comments_);
263
264    if (location != NULL) {
265      location->AttachComments(&leading, &trailing);
266    }
267    return true;
268  } else {
269    return false;
270  }
271}
272
273bool Parser::ConsumeEndOfDeclaration(const char* text,
274                                     const LocationRecorder* location) {
275  if (TryConsumeEndOfDeclaration(text, location)) {
276    return true;
277  } else {
278    AddError("Expected \"" + string(text) + "\".");
279    return false;
280  }
281}
282
283// -------------------------------------------------------------------
284
285void Parser::AddError(int line, int column, const string& error) {
286  if (error_collector_ != NULL) {
287    error_collector_->AddError(line, column, error);
288  }
289  had_errors_ = true;
290}
291
292void Parser::AddError(const string& error) {
293  AddError(input_->current().line, input_->current().column, error);
294}
295
296// -------------------------------------------------------------------
297
298Parser::LocationRecorder::LocationRecorder(Parser* parser)
299  : parser_(parser),
300    location_(parser_->source_code_info_->add_location()) {
301  location_->add_span(parser_->input_->current().line);
302  location_->add_span(parser_->input_->current().column);
303}
304
305Parser::LocationRecorder::LocationRecorder(const LocationRecorder& parent) {
306  Init(parent);
307}
308
309Parser::LocationRecorder::LocationRecorder(const LocationRecorder& parent,
310                                           int path1) {
311  Init(parent);
312  AddPath(path1);
313}
314
315Parser::LocationRecorder::LocationRecorder(const LocationRecorder& parent,
316                                           int path1, int path2) {
317  Init(parent);
318  AddPath(path1);
319  AddPath(path2);
320}
321
322void Parser::LocationRecorder::Init(const LocationRecorder& parent) {
323  parser_ = parent.parser_;
324  location_ = parser_->source_code_info_->add_location();
325  location_->mutable_path()->CopyFrom(parent.location_->path());
326
327  location_->add_span(parser_->input_->current().line);
328  location_->add_span(parser_->input_->current().column);
329}
330
331Parser::LocationRecorder::~LocationRecorder() {
332  if (location_->span_size() <= 2) {
333    EndAt(parser_->input_->previous());
334  }
335}
336
337void Parser::LocationRecorder::AddPath(int path_component) {
338  location_->add_path(path_component);
339}
340
341void Parser::LocationRecorder::StartAt(const io::Tokenizer::Token& token) {
342  location_->set_span(0, token.line);
343  location_->set_span(1, token.column);
344}
345
346void Parser::LocationRecorder::EndAt(const io::Tokenizer::Token& token) {
347  if (token.line != location_->span(0)) {
348    location_->add_span(token.line);
349  }
350  location_->add_span(token.end_column);
351}
352
353void Parser::LocationRecorder::RecordLegacyLocation(const Message* descriptor,
354    DescriptorPool::ErrorCollector::ErrorLocation location) {
355  if (parser_->source_location_table_ != NULL) {
356    parser_->source_location_table_->Add(
357        descriptor, location, location_->span(0), location_->span(1));
358  }
359}
360
361void Parser::LocationRecorder::AttachComments(
362    string* leading, string* trailing) const {
363  GOOGLE_CHECK(!location_->has_leading_comments());
364  GOOGLE_CHECK(!location_->has_trailing_comments());
365
366  if (!leading->empty()) {
367    location_->mutable_leading_comments()->swap(*leading);
368  }
369  if (!trailing->empty()) {
370    location_->mutable_trailing_comments()->swap(*trailing);
371  }
372}
373
374// -------------------------------------------------------------------
375
376void Parser::SkipStatement() {
377  while (true) {
378    if (AtEnd()) {
379      return;
380    } else if (LookingAtType(io::Tokenizer::TYPE_SYMBOL)) {
381      if (TryConsumeEndOfDeclaration(";", NULL)) {
382        return;
383      } else if (TryConsume("{")) {
384        SkipRestOfBlock();
385        return;
386      } else if (LookingAt("}")) {
387        return;
388      }
389    }
390    input_->Next();
391  }
392}
393
394void Parser::SkipRestOfBlock() {
395  while (true) {
396    if (AtEnd()) {
397      return;
398    } else if (LookingAtType(io::Tokenizer::TYPE_SYMBOL)) {
399      if (TryConsumeEndOfDeclaration("}", NULL)) {
400        return;
401      } else if (TryConsume("{")) {
402        SkipRestOfBlock();
403      }
404    }
405    input_->Next();
406  }
407}
408
409// ===================================================================
410
411bool Parser::Parse(io::Tokenizer* input, FileDescriptorProto* file) {
412  input_ = input;
413  had_errors_ = false;
414  syntax_identifier_.clear();
415
416  // Note that |file| could be NULL at this point if
417  // stop_after_syntax_identifier_ is true.  So, we conservatively allocate
418  // SourceCodeInfo on the stack, then swap it into the FileDescriptorProto
419  // later on.
420  SourceCodeInfo source_code_info;
421  source_code_info_ = &source_code_info;
422
423  if (LookingAtType(io::Tokenizer::TYPE_START)) {
424    // Advance to first token.
425    input_->NextWithComments(NULL, NULL, &upcoming_doc_comments_);
426  }
427
428  {
429    LocationRecorder root_location(this);
430
431    if (require_syntax_identifier_ || LookingAt("syntax")) {
432      if (!ParseSyntaxIdentifier()) {
433        // Don't attempt to parse the file if we didn't recognize the syntax
434        // identifier.
435        return false;
436      }
437    } else if (!stop_after_syntax_identifier_) {
438      syntax_identifier_ = "proto2";
439    }
440
441    if (stop_after_syntax_identifier_) return !had_errors_;
442
443    // Repeatedly parse statements until we reach the end of the file.
444    while (!AtEnd()) {
445      if (!ParseTopLevelStatement(file, root_location)) {
446        // This statement failed to parse.  Skip it, but keep looping to parse
447        // other statements.
448        SkipStatement();
449
450        if (LookingAt("}")) {
451          AddError("Unmatched \"}\".");
452          input_->NextWithComments(NULL, NULL, &upcoming_doc_comments_);
453        }
454      }
455    }
456  }
457
458  input_ = NULL;
459  source_code_info_ = NULL;
460  source_code_info.Swap(file->mutable_source_code_info());
461  return !had_errors_;
462}
463
464bool Parser::ParseSyntaxIdentifier() {
465  DO(Consume("syntax", "File must begin with 'syntax = \"proto2\";'."));
466  DO(Consume("="));
467  io::Tokenizer::Token syntax_token = input_->current();
468  string syntax;
469  DO(ConsumeString(&syntax, "Expected syntax identifier."));
470  DO(ConsumeEndOfDeclaration(";", NULL));
471
472  syntax_identifier_ = syntax;
473
474  if (syntax != "proto2" && !stop_after_syntax_identifier_) {
475    AddError(syntax_token.line, syntax_token.column,
476      "Unrecognized syntax identifier \"" + syntax + "\".  This parser "
477      "only recognizes \"proto2\".");
478    return false;
479  }
480
481  return true;
482}
483
484bool Parser::ParseTopLevelStatement(FileDescriptorProto* file,
485                                    const LocationRecorder& root_location) {
486  if (TryConsumeEndOfDeclaration(";", NULL)) {
487    // empty statement; ignore
488    return true;
489  } else if (LookingAt("message")) {
490    LocationRecorder location(root_location,
491      FileDescriptorProto::kMessageTypeFieldNumber, file->message_type_size());
492    return ParseMessageDefinition(file->add_message_type(), location);
493  } else if (LookingAt("enum")) {
494    LocationRecorder location(root_location,
495      FileDescriptorProto::kEnumTypeFieldNumber, file->enum_type_size());
496    return ParseEnumDefinition(file->add_enum_type(), location);
497  } else if (LookingAt("service")) {
498    LocationRecorder location(root_location,
499      FileDescriptorProto::kServiceFieldNumber, file->service_size());
500    return ParseServiceDefinition(file->add_service(), location);
501  } else if (LookingAt("extend")) {
502    LocationRecorder location(root_location,
503        FileDescriptorProto::kExtensionFieldNumber);
504    return ParseExtend(file->mutable_extension(),
505                       file->mutable_message_type(),
506                       root_location,
507                       FileDescriptorProto::kMessageTypeFieldNumber,
508                       location);
509  } else if (LookingAt("import")) {
510    return ParseImport(file->mutable_dependency(),
511                       file->mutable_public_dependency(),
512                       file->mutable_weak_dependency(),
513                       root_location);
514  } else if (LookingAt("package")) {
515    return ParsePackage(file, root_location);
516  } else if (LookingAt("option")) {
517    LocationRecorder location(root_location,
518        FileDescriptorProto::kOptionsFieldNumber);
519    return ParseOption(file->mutable_options(), location, OPTION_STATEMENT);
520  } else {
521    AddError("Expected top-level statement (e.g. \"message\").");
522    return false;
523  }
524}
525
526// -------------------------------------------------------------------
527// Messages
528
529bool Parser::ParseMessageDefinition(DescriptorProto* message,
530                                    const LocationRecorder& message_location) {
531  DO(Consume("message"));
532  {
533    LocationRecorder location(message_location,
534                              DescriptorProto::kNameFieldNumber);
535    location.RecordLegacyLocation(
536        message, DescriptorPool::ErrorCollector::NAME);
537    DO(ConsumeIdentifier(message->mutable_name(), "Expected message name."));
538  }
539  DO(ParseMessageBlock(message, message_location));
540  return true;
541}
542
543namespace {
544
545const int kMaxExtensionRangeSentinel = -1;
546
547bool IsMessageSetWireFormatMessage(const DescriptorProto& message) {
548  const MessageOptions& options = message.options();
549  for (int i = 0; i < options.uninterpreted_option_size(); ++i) {
550    const UninterpretedOption& uninterpreted = options.uninterpreted_option(i);
551    if (uninterpreted.name_size() == 1 &&
552        uninterpreted.name(0).name_part() == "message_set_wire_format" &&
553        uninterpreted.identifier_value() == "true") {
554      return true;
555    }
556  }
557  return false;
558}
559
560// Modifies any extension ranges that specified 'max' as the end of the
561// extension range, and sets them to the type-specific maximum. The actual max
562// tag number can only be determined after all options have been parsed.
563void AdjustExtensionRangesWithMaxEndNumber(DescriptorProto* message) {
564  const bool is_message_set = IsMessageSetWireFormatMessage(*message);
565  const int max_extension_number = is_message_set ?
566      kint32max :
567      FieldDescriptor::kMaxNumber + 1;
568  for (int i = 0; i < message->extension_range_size(); ++i) {
569    if (message->extension_range(i).end() == kMaxExtensionRangeSentinel) {
570      message->mutable_extension_range(i)->set_end(max_extension_number);
571    }
572  }
573}
574
575}  // namespace
576
577bool Parser::ParseMessageBlock(DescriptorProto* message,
578                               const LocationRecorder& message_location) {
579  DO(ConsumeEndOfDeclaration("{", &message_location));
580
581  while (!TryConsumeEndOfDeclaration("}", NULL)) {
582    if (AtEnd()) {
583      AddError("Reached end of input in message definition (missing '}').");
584      return false;
585    }
586
587    if (!ParseMessageStatement(message, message_location)) {
588      // This statement failed to parse.  Skip it, but keep looping to parse
589      // other statements.
590      SkipStatement();
591    }
592  }
593
594  if (message->extension_range_size() > 0) {
595    AdjustExtensionRangesWithMaxEndNumber(message);
596  }
597  return true;
598}
599
600bool Parser::ParseMessageStatement(DescriptorProto* message,
601                                   const LocationRecorder& message_location) {
602  if (TryConsumeEndOfDeclaration(";", NULL)) {
603    // empty statement; ignore
604    return true;
605  } else if (LookingAt("message")) {
606    LocationRecorder location(message_location,
607                              DescriptorProto::kNestedTypeFieldNumber,
608                              message->nested_type_size());
609    return ParseMessageDefinition(message->add_nested_type(), location);
610  } else if (LookingAt("enum")) {
611    LocationRecorder location(message_location,
612                              DescriptorProto::kEnumTypeFieldNumber,
613                              message->enum_type_size());
614    return ParseEnumDefinition(message->add_enum_type(), location);
615  } else if (LookingAt("extensions")) {
616    LocationRecorder location(message_location,
617                              DescriptorProto::kExtensionRangeFieldNumber);
618    return ParseExtensions(message, location);
619  } else if (LookingAt("extend")) {
620    LocationRecorder location(message_location,
621                              DescriptorProto::kExtensionFieldNumber);
622    return ParseExtend(message->mutable_extension(),
623                       message->mutable_nested_type(),
624                       message_location,
625                       DescriptorProto::kNestedTypeFieldNumber,
626                       location);
627  } else if (LookingAt("option")) {
628    LocationRecorder location(message_location,
629                              DescriptorProto::kOptionsFieldNumber);
630    return ParseOption(message->mutable_options(), location, OPTION_STATEMENT);
631  } else {
632    LocationRecorder location(message_location,
633                              DescriptorProto::kFieldFieldNumber,
634                              message->field_size());
635    return ParseMessageField(message->add_field(),
636                             message->mutable_nested_type(),
637                             message_location,
638                             DescriptorProto::kNestedTypeFieldNumber,
639                             location);
640  }
641}
642
643bool Parser::ParseMessageField(FieldDescriptorProto* field,
644                               RepeatedPtrField<DescriptorProto>* messages,
645                               const LocationRecorder& parent_location,
646                               int location_field_number_for_nested_type,
647                               const LocationRecorder& field_location) {
648  // Parse label and type.
649  io::Tokenizer::Token label_token = input_->current();
650  {
651    LocationRecorder location(field_location,
652                              FieldDescriptorProto::kLabelFieldNumber);
653    FieldDescriptorProto::Label label;
654    DO(ParseLabel(&label));
655    field->set_label(label);
656  }
657
658  {
659    LocationRecorder location(field_location);  // add path later
660    location.RecordLegacyLocation(field, DescriptorPool::ErrorCollector::TYPE);
661
662    FieldDescriptorProto::Type type = FieldDescriptorProto::TYPE_INT32;
663    string type_name;
664    DO(ParseType(&type, &type_name));
665    if (type_name.empty()) {
666      location.AddPath(FieldDescriptorProto::kTypeFieldNumber);
667      field->set_type(type);
668    } else {
669      location.AddPath(FieldDescriptorProto::kTypeNameFieldNumber);
670      field->set_type_name(type_name);
671    }
672  }
673
674  // Parse name and '='.
675  io::Tokenizer::Token name_token = input_->current();
676  {
677    LocationRecorder location(field_location,
678                              FieldDescriptorProto::kNameFieldNumber);
679    location.RecordLegacyLocation(field, DescriptorPool::ErrorCollector::NAME);
680    DO(ConsumeIdentifier(field->mutable_name(), "Expected field name."));
681  }
682  DO(Consume("=", "Missing field number."));
683
684  // Parse field number.
685  {
686    LocationRecorder location(field_location,
687                              FieldDescriptorProto::kNumberFieldNumber);
688    location.RecordLegacyLocation(
689        field, DescriptorPool::ErrorCollector::NUMBER);
690    int number;
691    DO(ConsumeInteger(&number, "Expected field number."));
692    field->set_number(number);
693  }
694
695  // Parse options.
696  DO(ParseFieldOptions(field, field_location));
697
698  // Deal with groups.
699  if (field->has_type() && field->type() == FieldDescriptorProto::TYPE_GROUP) {
700    // Awkward:  Since a group declares both a message type and a field, we
701    //   have to create overlapping locations.
702    LocationRecorder group_location(parent_location);
703    group_location.StartAt(label_token);
704    group_location.AddPath(location_field_number_for_nested_type);
705    group_location.AddPath(messages->size());
706
707    DescriptorProto* group = messages->Add();
708    group->set_name(field->name());
709
710    // Record name location to match the field name's location.
711    {
712      LocationRecorder location(group_location,
713                                DescriptorProto::kNameFieldNumber);
714      location.StartAt(name_token);
715      location.EndAt(name_token);
716      location.RecordLegacyLocation(
717          group, DescriptorPool::ErrorCollector::NAME);
718    }
719
720    // The field's type_name also comes from the name.  Confusing!
721    {
722      LocationRecorder location(field_location,
723                                FieldDescriptorProto::kTypeNameFieldNumber);
724      location.StartAt(name_token);
725      location.EndAt(name_token);
726    }
727
728    // As a hack for backwards-compatibility, we force the group name to start
729    // with a capital letter and lower-case the field name.  New code should
730    // not use groups; it should use nested messages.
731    if (group->name()[0] < 'A' || 'Z' < group->name()[0]) {
732      AddError(name_token.line, name_token.column,
733        "Group names must start with a capital letter.");
734    }
735    LowerString(field->mutable_name());
736
737    field->set_type_name(group->name());
738    if (LookingAt("{")) {
739      DO(ParseMessageBlock(group, group_location));
740    } else {
741      AddError("Missing group body.");
742      return false;
743    }
744  } else {
745    DO(ConsumeEndOfDeclaration(";", &field_location));
746  }
747
748  return true;
749}
750
751bool Parser::ParseFieldOptions(FieldDescriptorProto* field,
752                               const LocationRecorder& field_location) {
753  if (!LookingAt("[")) return true;
754
755  LocationRecorder location(field_location,
756                            FieldDescriptorProto::kOptionsFieldNumber);
757
758  DO(Consume("["));
759
760  // Parse field options.
761  do {
762    if (LookingAt("default")) {
763      // We intentionally pass field_location rather than location here, since
764      // the default value is not actually an option.
765      DO(ParseDefaultAssignment(field, field_location));
766    } else {
767      DO(ParseOption(field->mutable_options(), location, OPTION_ASSIGNMENT));
768    }
769  } while (TryConsume(","));
770
771  DO(Consume("]"));
772  return true;
773}
774
775bool Parser::ParseDefaultAssignment(FieldDescriptorProto* field,
776                                    const LocationRecorder& field_location) {
777  if (field->has_default_value()) {
778    AddError("Already set option \"default\".");
779    field->clear_default_value();
780  }
781
782  DO(Consume("default"));
783  DO(Consume("="));
784
785  LocationRecorder location(field_location,
786                            FieldDescriptorProto::kDefaultValueFieldNumber);
787  location.RecordLegacyLocation(
788      field, DescriptorPool::ErrorCollector::DEFAULT_VALUE);
789  string* default_value = field->mutable_default_value();
790
791  if (!field->has_type()) {
792    // The field has a type name, but we don't know if it is a message or an
793    // enum yet.  Assume an enum for now.
794    DO(ConsumeIdentifier(default_value, "Expected identifier."));
795    return true;
796  }
797
798  switch (field->type()) {
799    case FieldDescriptorProto::TYPE_INT32:
800    case FieldDescriptorProto::TYPE_INT64:
801    case FieldDescriptorProto::TYPE_SINT32:
802    case FieldDescriptorProto::TYPE_SINT64:
803    case FieldDescriptorProto::TYPE_SFIXED32:
804    case FieldDescriptorProto::TYPE_SFIXED64: {
805      uint64 max_value = kint64max;
806      if (field->type() == FieldDescriptorProto::TYPE_INT32 ||
807          field->type() == FieldDescriptorProto::TYPE_SINT32 ||
808          field->type() == FieldDescriptorProto::TYPE_SFIXED32) {
809        max_value = kint32max;
810      }
811
812      // These types can be negative.
813      if (TryConsume("-")) {
814        default_value->append("-");
815        // Two's complement always has one more negative value than positive.
816        ++max_value;
817      }
818      // Parse the integer to verify that it is not out-of-range.
819      uint64 value;
820      DO(ConsumeInteger64(max_value, &value, "Expected integer."));
821      // And stringify it again.
822      default_value->append(SimpleItoa(value));
823      break;
824    }
825
826    case FieldDescriptorProto::TYPE_UINT32:
827    case FieldDescriptorProto::TYPE_UINT64:
828    case FieldDescriptorProto::TYPE_FIXED32:
829    case FieldDescriptorProto::TYPE_FIXED64: {
830      uint64 max_value = kuint64max;
831      if (field->type() == FieldDescriptorProto::TYPE_UINT32 ||
832          field->type() == FieldDescriptorProto::TYPE_FIXED32) {
833        max_value = kuint32max;
834      }
835
836      // Numeric, not negative.
837      if (TryConsume("-")) {
838        AddError("Unsigned field can't have negative default value.");
839      }
840      // Parse the integer to verify that it is not out-of-range.
841      uint64 value;
842      DO(ConsumeInteger64(max_value, &value, "Expected integer."));
843      // And stringify it again.
844      default_value->append(SimpleItoa(value));
845      break;
846    }
847
848    case FieldDescriptorProto::TYPE_FLOAT:
849    case FieldDescriptorProto::TYPE_DOUBLE:
850      // These types can be negative.
851      if (TryConsume("-")) {
852        default_value->append("-");
853      }
854      // Parse the integer because we have to convert hex integers to decimal
855      // floats.
856      double value;
857      DO(ConsumeNumber(&value, "Expected number."));
858      // And stringify it again.
859      default_value->append(SimpleDtoa(value));
860      break;
861
862    case FieldDescriptorProto::TYPE_BOOL:
863      if (TryConsume("true")) {
864        default_value->assign("true");
865      } else if (TryConsume("false")) {
866        default_value->assign("false");
867      } else {
868        AddError("Expected \"true\" or \"false\".");
869        return false;
870      }
871      break;
872
873    case FieldDescriptorProto::TYPE_STRING:
874      DO(ConsumeString(default_value, "Expected string."));
875      break;
876
877    case FieldDescriptorProto::TYPE_BYTES:
878      DO(ConsumeString(default_value, "Expected string."));
879      *default_value = CEscape(*default_value);
880      break;
881
882    case FieldDescriptorProto::TYPE_ENUM:
883      DO(ConsumeIdentifier(default_value, "Expected identifier."));
884      break;
885
886    case FieldDescriptorProto::TYPE_MESSAGE:
887    case FieldDescriptorProto::TYPE_GROUP:
888      AddError("Messages can't have default values.");
889      return false;
890  }
891
892  return true;
893}
894
895bool Parser::ParseOptionNamePart(UninterpretedOption* uninterpreted_option,
896                                 const LocationRecorder& part_location) {
897  UninterpretedOption::NamePart* name = uninterpreted_option->add_name();
898  string identifier;  // We parse identifiers into this string.
899  if (LookingAt("(")) {  // This is an extension.
900    DO(Consume("("));
901
902    {
903      LocationRecorder location(
904          part_location, UninterpretedOption::NamePart::kNamePartFieldNumber);
905      // An extension name consists of dot-separated identifiers, and may begin
906      // with a dot.
907      if (LookingAtType(io::Tokenizer::TYPE_IDENTIFIER)) {
908        DO(ConsumeIdentifier(&identifier, "Expected identifier."));
909        name->mutable_name_part()->append(identifier);
910      }
911      while (LookingAt(".")) {
912        DO(Consume("."));
913        name->mutable_name_part()->append(".");
914        DO(ConsumeIdentifier(&identifier, "Expected identifier."));
915        name->mutable_name_part()->append(identifier);
916      }
917    }
918
919    DO(Consume(")"));
920    name->set_is_extension(true);
921  } else {  // This is a regular field.
922    LocationRecorder location(
923        part_location, UninterpretedOption::NamePart::kNamePartFieldNumber);
924    DO(ConsumeIdentifier(&identifier, "Expected identifier."));
925    name->mutable_name_part()->append(identifier);
926    name->set_is_extension(false);
927  }
928  return true;
929}
930
931bool Parser::ParseUninterpretedBlock(string* value) {
932  // Note that enclosing braces are not added to *value.
933  // We do NOT use ConsumeEndOfStatement for this brace because it's delimiting
934  // an expression, not a block of statements.
935  DO(Consume("{"));
936  int brace_depth = 1;
937  while (!AtEnd()) {
938    if (LookingAt("{")) {
939      brace_depth++;
940    } else if (LookingAt("}")) {
941      brace_depth--;
942      if (brace_depth == 0) {
943        input_->Next();
944        return true;
945      }
946    }
947    // TODO(sanjay): Interpret line/column numbers to preserve formatting
948    if (!value->empty()) value->push_back(' ');
949    value->append(input_->current().text);
950    input_->Next();
951  }
952  AddError("Unexpected end of stream while parsing aggregate value.");
953  return false;
954}
955
956// We don't interpret the option here. Instead we store it in an
957// UninterpretedOption, to be interpreted later.
958bool Parser::ParseOption(Message* options,
959                         const LocationRecorder& options_location,
960                         OptionStyle style) {
961  // Create an entry in the uninterpreted_option field.
962  const FieldDescriptor* uninterpreted_option_field = options->GetDescriptor()->
963      FindFieldByName("uninterpreted_option");
964  GOOGLE_CHECK(uninterpreted_option_field != NULL)
965      << "No field named \"uninterpreted_option\" in the Options proto.";
966
967  const Reflection* reflection = options->GetReflection();
968
969  LocationRecorder location(
970      options_location, uninterpreted_option_field->number(),
971      reflection->FieldSize(*options, uninterpreted_option_field));
972
973  if (style == OPTION_STATEMENT) {
974    DO(Consume("option"));
975  }
976
977  UninterpretedOption* uninterpreted_option = down_cast<UninterpretedOption*>(
978      options->GetReflection()->AddMessage(options,
979                                           uninterpreted_option_field));
980
981  // Parse dot-separated name.
982  {
983    LocationRecorder name_location(location,
984                                   UninterpretedOption::kNameFieldNumber);
985    name_location.RecordLegacyLocation(
986        uninterpreted_option, DescriptorPool::ErrorCollector::OPTION_NAME);
987
988    {
989      LocationRecorder part_location(name_location,
990                                     uninterpreted_option->name_size());
991      DO(ParseOptionNamePart(uninterpreted_option, part_location));
992    }
993
994    while (LookingAt(".")) {
995      DO(Consume("."));
996      LocationRecorder part_location(name_location,
997                                     uninterpreted_option->name_size());
998      DO(ParseOptionNamePart(uninterpreted_option, part_location));
999    }
1000  }
1001
1002  DO(Consume("="));
1003
1004  {
1005    LocationRecorder value_location(location);
1006    value_location.RecordLegacyLocation(
1007        uninterpreted_option, DescriptorPool::ErrorCollector::OPTION_VALUE);
1008
1009    // All values are a single token, except for negative numbers, which consist
1010    // of a single '-' symbol, followed by a positive number.
1011    bool is_negative = TryConsume("-");
1012
1013    switch (input_->current().type) {
1014      case io::Tokenizer::TYPE_START:
1015        GOOGLE_LOG(FATAL) << "Trying to read value before any tokens have been read.";
1016        return false;
1017
1018      case io::Tokenizer::TYPE_END:
1019        AddError("Unexpected end of stream while parsing option value.");
1020        return false;
1021
1022      case io::Tokenizer::TYPE_IDENTIFIER: {
1023        value_location.AddPath(
1024            UninterpretedOption::kIdentifierValueFieldNumber);
1025        if (is_negative) {
1026          AddError("Invalid '-' symbol before identifier.");
1027          return false;
1028        }
1029        string value;
1030        DO(ConsumeIdentifier(&value, "Expected identifier."));
1031        uninterpreted_option->set_identifier_value(value);
1032        break;
1033      }
1034
1035      case io::Tokenizer::TYPE_INTEGER: {
1036        uint64 value;
1037        uint64 max_value =
1038            is_negative ? static_cast<uint64>(kint64max) + 1 : kuint64max;
1039        DO(ConsumeInteger64(max_value, &value, "Expected integer."));
1040        if (is_negative) {
1041          value_location.AddPath(
1042              UninterpretedOption::kNegativeIntValueFieldNumber);
1043          uninterpreted_option->set_negative_int_value(
1044              -static_cast<int64>(value));
1045        } else {
1046          value_location.AddPath(
1047              UninterpretedOption::kPositiveIntValueFieldNumber);
1048          uninterpreted_option->set_positive_int_value(value);
1049        }
1050        break;
1051      }
1052
1053      case io::Tokenizer::TYPE_FLOAT: {
1054        value_location.AddPath(UninterpretedOption::kDoubleValueFieldNumber);
1055        double value;
1056        DO(ConsumeNumber(&value, "Expected number."));
1057        uninterpreted_option->set_double_value(is_negative ? -value : value);
1058        break;
1059      }
1060
1061      case io::Tokenizer::TYPE_STRING: {
1062        value_location.AddPath(UninterpretedOption::kStringValueFieldNumber);
1063        if (is_negative) {
1064          AddError("Invalid '-' symbol before string.");
1065          return false;
1066        }
1067        string value;
1068        DO(ConsumeString(&value, "Expected string."));
1069        uninterpreted_option->set_string_value(value);
1070        break;
1071      }
1072
1073      case io::Tokenizer::TYPE_SYMBOL:
1074        if (LookingAt("{")) {
1075          value_location.AddPath(
1076              UninterpretedOption::kAggregateValueFieldNumber);
1077          DO(ParseUninterpretedBlock(
1078              uninterpreted_option->mutable_aggregate_value()));
1079        } else {
1080          AddError("Expected option value.");
1081          return false;
1082        }
1083        break;
1084    }
1085  }
1086
1087  if (style == OPTION_STATEMENT) {
1088    DO(ConsumeEndOfDeclaration(";", &location));
1089  }
1090
1091  return true;
1092}
1093
1094bool Parser::ParseExtensions(DescriptorProto* message,
1095                             const LocationRecorder& extensions_location) {
1096  // Parse the declaration.
1097  DO(Consume("extensions"));
1098
1099  do {
1100    // Note that kExtensionRangeFieldNumber was already pushed by the parent.
1101    LocationRecorder location(extensions_location,
1102                              message->extension_range_size());
1103
1104    DescriptorProto::ExtensionRange* range = message->add_extension_range();
1105    location.RecordLegacyLocation(
1106        range, DescriptorPool::ErrorCollector::NUMBER);
1107
1108    int start, end;
1109    io::Tokenizer::Token start_token;
1110
1111    {
1112      LocationRecorder start_location(
1113          location, DescriptorProto::ExtensionRange::kStartFieldNumber);
1114      start_token = input_->current();
1115      DO(ConsumeInteger(&start, "Expected field number range."));
1116    }
1117
1118    if (TryConsume("to")) {
1119      LocationRecorder end_location(
1120          location, DescriptorProto::ExtensionRange::kEndFieldNumber);
1121      if (TryConsume("max")) {
1122        // Set to the sentinel value - 1 since we increment the value below.
1123        // The actual value of the end of the range should be set with
1124        // AdjustExtensionRangesWithMaxEndNumber.
1125        end = kMaxExtensionRangeSentinel - 1;
1126      } else {
1127        DO(ConsumeInteger(&end, "Expected integer."));
1128      }
1129    } else {
1130      LocationRecorder end_location(
1131          location, DescriptorProto::ExtensionRange::kEndFieldNumber);
1132      end_location.StartAt(start_token);
1133      end_location.EndAt(start_token);
1134      end = start;
1135    }
1136
1137    // Users like to specify inclusive ranges, but in code we like the end
1138    // number to be exclusive.
1139    ++end;
1140
1141    range->set_start(start);
1142    range->set_end(end);
1143  } while (TryConsume(","));
1144
1145  DO(ConsumeEndOfDeclaration(";", &extensions_location));
1146  return true;
1147}
1148
1149bool Parser::ParseExtend(RepeatedPtrField<FieldDescriptorProto>* extensions,
1150                         RepeatedPtrField<DescriptorProto>* messages,
1151                         const LocationRecorder& parent_location,
1152                         int location_field_number_for_nested_type,
1153                         const LocationRecorder& extend_location) {
1154  DO(Consume("extend"));
1155
1156  // Parse the extendee type.
1157  io::Tokenizer::Token extendee_start = input_->current();
1158  string extendee;
1159  DO(ParseUserDefinedType(&extendee));
1160  io::Tokenizer::Token extendee_end = input_->previous();
1161
1162  // Parse the block.
1163  DO(ConsumeEndOfDeclaration("{", &extend_location));
1164
1165  bool is_first = true;
1166
1167  do {
1168    if (AtEnd()) {
1169      AddError("Reached end of input in extend definition (missing '}').");
1170      return false;
1171    }
1172
1173    // Note that kExtensionFieldNumber was already pushed by the parent.
1174    LocationRecorder location(extend_location, extensions->size());
1175
1176    FieldDescriptorProto* field = extensions->Add();
1177
1178    {
1179      LocationRecorder extendee_location(
1180          location, FieldDescriptorProto::kExtendeeFieldNumber);
1181      extendee_location.StartAt(extendee_start);
1182      extendee_location.EndAt(extendee_end);
1183
1184      if (is_first) {
1185        extendee_location.RecordLegacyLocation(
1186            field, DescriptorPool::ErrorCollector::EXTENDEE);
1187        is_first = false;
1188      }
1189    }
1190
1191    field->set_extendee(extendee);
1192
1193    if (!ParseMessageField(field, messages, parent_location,
1194                           location_field_number_for_nested_type,
1195                           location)) {
1196      // This statement failed to parse.  Skip it, but keep looping to parse
1197      // other statements.
1198      SkipStatement();
1199    }
1200  } while (!TryConsumeEndOfDeclaration("}", NULL));
1201
1202  return true;
1203}
1204
1205// -------------------------------------------------------------------
1206// Enums
1207
1208bool Parser::ParseEnumDefinition(EnumDescriptorProto* enum_type,
1209                                 const LocationRecorder& enum_location) {
1210  DO(Consume("enum"));
1211
1212  {
1213    LocationRecorder location(enum_location,
1214                              EnumDescriptorProto::kNameFieldNumber);
1215    location.RecordLegacyLocation(
1216        enum_type, DescriptorPool::ErrorCollector::NAME);
1217    DO(ConsumeIdentifier(enum_type->mutable_name(), "Expected enum name."));
1218  }
1219
1220  DO(ParseEnumBlock(enum_type, enum_location));
1221  return true;
1222}
1223
1224bool Parser::ParseEnumBlock(EnumDescriptorProto* enum_type,
1225                            const LocationRecorder& enum_location) {
1226  DO(ConsumeEndOfDeclaration("{", &enum_location));
1227
1228  while (!TryConsumeEndOfDeclaration("}", NULL)) {
1229    if (AtEnd()) {
1230      AddError("Reached end of input in enum definition (missing '}').");
1231      return false;
1232    }
1233
1234    if (!ParseEnumStatement(enum_type, enum_location)) {
1235      // This statement failed to parse.  Skip it, but keep looping to parse
1236      // other statements.
1237      SkipStatement();
1238    }
1239  }
1240
1241  return true;
1242}
1243
1244bool Parser::ParseEnumStatement(EnumDescriptorProto* enum_type,
1245                                const LocationRecorder& enum_location) {
1246  if (TryConsumeEndOfDeclaration(";", NULL)) {
1247    // empty statement; ignore
1248    return true;
1249  } else if (LookingAt("option")) {
1250    LocationRecorder location(enum_location,
1251                              EnumDescriptorProto::kOptionsFieldNumber);
1252    return ParseOption(enum_type->mutable_options(), location,
1253                       OPTION_STATEMENT);
1254  } else {
1255    LocationRecorder location(enum_location,
1256        EnumDescriptorProto::kValueFieldNumber, enum_type->value_size());
1257    return ParseEnumConstant(enum_type->add_value(), location);
1258  }
1259}
1260
1261bool Parser::ParseEnumConstant(EnumValueDescriptorProto* enum_value,
1262                               const LocationRecorder& enum_value_location) {
1263  // Parse name.
1264  {
1265    LocationRecorder location(enum_value_location,
1266                              EnumValueDescriptorProto::kNameFieldNumber);
1267    location.RecordLegacyLocation(
1268        enum_value, DescriptorPool::ErrorCollector::NAME);
1269    DO(ConsumeIdentifier(enum_value->mutable_name(),
1270                         "Expected enum constant name."));
1271  }
1272
1273  DO(Consume("=", "Missing numeric value for enum constant."));
1274
1275  // Parse value.
1276  {
1277    LocationRecorder location(
1278        enum_value_location, EnumValueDescriptorProto::kNumberFieldNumber);
1279    location.RecordLegacyLocation(
1280        enum_value, DescriptorPool::ErrorCollector::NUMBER);
1281
1282    int number;
1283    DO(ConsumeSignedInteger(&number, "Expected integer."));
1284    enum_value->set_number(number);
1285  }
1286
1287  DO(ParseEnumConstantOptions(enum_value, enum_value_location));
1288
1289  DO(ConsumeEndOfDeclaration(";", &enum_value_location));
1290
1291  return true;
1292}
1293
1294bool Parser::ParseEnumConstantOptions(
1295    EnumValueDescriptorProto* value,
1296    const LocationRecorder& enum_value_location) {
1297  if (!LookingAt("[")) return true;
1298
1299  LocationRecorder location(
1300      enum_value_location, EnumValueDescriptorProto::kOptionsFieldNumber);
1301
1302  DO(Consume("["));
1303
1304  do {
1305    DO(ParseOption(value->mutable_options(), location, OPTION_ASSIGNMENT));
1306  } while (TryConsume(","));
1307
1308  DO(Consume("]"));
1309  return true;
1310}
1311
1312// -------------------------------------------------------------------
1313// Services
1314
1315bool Parser::ParseServiceDefinition(ServiceDescriptorProto* service,
1316                                    const LocationRecorder& service_location) {
1317  DO(Consume("service"));
1318
1319  {
1320    LocationRecorder location(service_location,
1321                              ServiceDescriptorProto::kNameFieldNumber);
1322    location.RecordLegacyLocation(
1323        service, DescriptorPool::ErrorCollector::NAME);
1324    DO(ConsumeIdentifier(service->mutable_name(), "Expected service name."));
1325  }
1326
1327  DO(ParseServiceBlock(service, service_location));
1328  return true;
1329}
1330
1331bool Parser::ParseServiceBlock(ServiceDescriptorProto* service,
1332                               const LocationRecorder& service_location) {
1333  DO(ConsumeEndOfDeclaration("{", &service_location));
1334
1335  while (!TryConsumeEndOfDeclaration("}", NULL)) {
1336    if (AtEnd()) {
1337      AddError("Reached end of input in service definition (missing '}').");
1338      return false;
1339    }
1340
1341    if (!ParseServiceStatement(service, service_location)) {
1342      // This statement failed to parse.  Skip it, but keep looping to parse
1343      // other statements.
1344      SkipStatement();
1345    }
1346  }
1347
1348  return true;
1349}
1350
1351bool Parser::ParseServiceStatement(ServiceDescriptorProto* service,
1352                                   const LocationRecorder& service_location) {
1353  if (TryConsumeEndOfDeclaration(";", NULL)) {
1354    // empty statement; ignore
1355    return true;
1356  } else if (LookingAt("option")) {
1357    LocationRecorder location(
1358        service_location, ServiceDescriptorProto::kOptionsFieldNumber);
1359    return ParseOption(service->mutable_options(), location, OPTION_STATEMENT);
1360  } else {
1361    LocationRecorder location(service_location,
1362        ServiceDescriptorProto::kMethodFieldNumber, service->method_size());
1363    return ParseServiceMethod(service->add_method(), location);
1364  }
1365}
1366
1367bool Parser::ParseServiceMethod(MethodDescriptorProto* method,
1368                                const LocationRecorder& method_location) {
1369  DO(Consume("rpc"));
1370
1371  {
1372    LocationRecorder location(method_location,
1373                              MethodDescriptorProto::kNameFieldNumber);
1374    location.RecordLegacyLocation(
1375        method, DescriptorPool::ErrorCollector::NAME);
1376    DO(ConsumeIdentifier(method->mutable_name(), "Expected method name."));
1377  }
1378
1379  // Parse input type.
1380  DO(Consume("("));
1381  {
1382    LocationRecorder location(method_location,
1383                              MethodDescriptorProto::kInputTypeFieldNumber);
1384    location.RecordLegacyLocation(
1385        method, DescriptorPool::ErrorCollector::INPUT_TYPE);
1386    DO(ParseUserDefinedType(method->mutable_input_type()));
1387  }
1388  DO(Consume(")"));
1389
1390  // Parse output type.
1391  DO(Consume("returns"));
1392  DO(Consume("("));
1393  {
1394    LocationRecorder location(method_location,
1395                              MethodDescriptorProto::kOutputTypeFieldNumber);
1396    location.RecordLegacyLocation(
1397        method, DescriptorPool::ErrorCollector::OUTPUT_TYPE);
1398    DO(ParseUserDefinedType(method->mutable_output_type()));
1399  }
1400  DO(Consume(")"));
1401
1402  if (LookingAt("{")) {
1403    // Options!
1404    DO(ParseOptions(method_location,
1405                    MethodDescriptorProto::kOptionsFieldNumber,
1406                    method->mutable_options()));
1407  } else {
1408    DO(ConsumeEndOfDeclaration(";", &method_location));
1409  }
1410
1411  return true;
1412}
1413
1414
1415bool Parser::ParseOptions(const LocationRecorder& parent_location,
1416                          const int optionsFieldNumber,
1417                          Message* mutable_options) {
1418  // Options!
1419  ConsumeEndOfDeclaration("{", &parent_location);
1420  while (!TryConsumeEndOfDeclaration("}", NULL)) {
1421    if (AtEnd()) {
1422      AddError("Reached end of input in method options (missing '}').");
1423      return false;
1424    }
1425
1426    if (TryConsumeEndOfDeclaration(";", NULL)) {
1427      // empty statement; ignore
1428    } else {
1429      LocationRecorder location(parent_location,
1430                                optionsFieldNumber);
1431      if (!ParseOption(mutable_options, location, OPTION_STATEMENT)) {
1432        // This statement failed to parse.  Skip it, but keep looping to
1433        // parse other statements.
1434        SkipStatement();
1435      }
1436    }
1437  }
1438
1439  return true;
1440}
1441
1442// -------------------------------------------------------------------
1443
1444bool Parser::ParseLabel(FieldDescriptorProto::Label* label) {
1445  if (TryConsume("optional")) {
1446    *label = FieldDescriptorProto::LABEL_OPTIONAL;
1447    return true;
1448  } else if (TryConsume("repeated")) {
1449    *label = FieldDescriptorProto::LABEL_REPEATED;
1450    return true;
1451  } else if (TryConsume("required")) {
1452    *label = FieldDescriptorProto::LABEL_REQUIRED;
1453    return true;
1454  } else {
1455    AddError("Expected \"required\", \"optional\", or \"repeated\".");
1456    // We can actually reasonably recover here by just assuming the user
1457    // forgot the label altogether.
1458    *label = FieldDescriptorProto::LABEL_OPTIONAL;
1459    return true;
1460  }
1461}
1462
1463bool Parser::ParseType(FieldDescriptorProto::Type* type,
1464                       string* type_name) {
1465  TypeNameMap::const_iterator iter = kTypeNames.find(input_->current().text);
1466  if (iter != kTypeNames.end()) {
1467    *type = iter->second;
1468    input_->Next();
1469  } else {
1470    DO(ParseUserDefinedType(type_name));
1471  }
1472  return true;
1473}
1474
1475bool Parser::ParseUserDefinedType(string* type_name) {
1476  type_name->clear();
1477
1478  TypeNameMap::const_iterator iter = kTypeNames.find(input_->current().text);
1479  if (iter != kTypeNames.end()) {
1480    // Note:  The only place enum types are allowed is for field types, but
1481    //   if we are parsing a field type then we would not get here because
1482    //   primitives are allowed there as well.  So this error message doesn't
1483    //   need to account for enums.
1484    AddError("Expected message type.");
1485
1486    // Pretend to accept this type so that we can go on parsing.
1487    *type_name = input_->current().text;
1488    input_->Next();
1489    return true;
1490  }
1491
1492  // A leading "." means the name is fully-qualified.
1493  if (TryConsume(".")) type_name->append(".");
1494
1495  // Consume the first part of the name.
1496  string identifier;
1497  DO(ConsumeIdentifier(&identifier, "Expected type name."));
1498  type_name->append(identifier);
1499
1500  // Consume more parts.
1501  while (TryConsume(".")) {
1502    type_name->append(".");
1503    DO(ConsumeIdentifier(&identifier, "Expected identifier."));
1504    type_name->append(identifier);
1505  }
1506
1507  return true;
1508}
1509
1510// ===================================================================
1511
1512bool Parser::ParsePackage(FileDescriptorProto* file,
1513                          const LocationRecorder& root_location) {
1514  if (file->has_package()) {
1515    AddError("Multiple package definitions.");
1516    // Don't append the new package to the old one.  Just replace it.  Not
1517    // that it really matters since this is an error anyway.
1518    file->clear_package();
1519  }
1520
1521  DO(Consume("package"));
1522
1523  {
1524    LocationRecorder location(root_location,
1525                              FileDescriptorProto::kPackageFieldNumber);
1526    location.RecordLegacyLocation(file, DescriptorPool::ErrorCollector::NAME);
1527
1528    while (true) {
1529      string identifier;
1530      DO(ConsumeIdentifier(&identifier, "Expected identifier."));
1531      file->mutable_package()->append(identifier);
1532      if (!TryConsume(".")) break;
1533      file->mutable_package()->append(".");
1534    }
1535
1536    location.EndAt(input_->previous());
1537
1538    DO(ConsumeEndOfDeclaration(";", &location));
1539  }
1540
1541  return true;
1542}
1543
1544bool Parser::ParseImport(RepeatedPtrField<string>* dependency,
1545                         RepeatedField<int32>* public_dependency,
1546                         RepeatedField<int32>* weak_dependency,
1547                         const LocationRecorder& root_location) {
1548  DO(Consume("import"));
1549  if (LookingAt("public")) {
1550    LocationRecorder location(
1551        root_location, FileDescriptorProto::kPublicDependencyFieldNumber,
1552        public_dependency->size());
1553    DO(Consume("public"));
1554    *public_dependency->Add() = dependency->size();
1555  } else if (LookingAt("weak")) {
1556    LocationRecorder location(
1557        root_location, FileDescriptorProto::kWeakDependencyFieldNumber,
1558        weak_dependency->size());
1559    DO(Consume("weak"));
1560    *weak_dependency->Add() = dependency->size();
1561  }
1562  {
1563    LocationRecorder location(root_location,
1564                              FileDescriptorProto::kDependencyFieldNumber,
1565                              dependency->size());
1566    DO(ConsumeString(dependency->Add(),
1567      "Expected a string naming the file to import."));
1568
1569    location.EndAt(input_->previous());
1570
1571    DO(ConsumeEndOfDeclaration(";", &location));
1572  }
1573  return true;
1574}
1575
1576// ===================================================================
1577
1578SourceLocationTable::SourceLocationTable() {}
1579SourceLocationTable::~SourceLocationTable() {}
1580
1581bool SourceLocationTable::Find(
1582    const Message* descriptor,
1583    DescriptorPool::ErrorCollector::ErrorLocation location,
1584    int* line, int* column) const {
1585  const pair<int, int>* result =
1586    FindOrNull(location_map_, make_pair(descriptor, location));
1587  if (result == NULL) {
1588    *line   = -1;
1589    *column = 0;
1590    return false;
1591  } else {
1592    *line   = result->first;
1593    *column = result->second;
1594    return true;
1595  }
1596}
1597
1598void SourceLocationTable::Add(
1599    const Message* descriptor,
1600    DescriptorPool::ErrorCollector::ErrorLocation location,
1601    int line, int column) {
1602  location_map_[make_pair(descriptor, location)] = make_pair(line, column);
1603}
1604
1605void SourceLocationTable::Clear() {
1606  location_map_.clear();
1607}
1608
1609}  // namespace compiler
1610}  // namespace protobuf
1611}  // namespace google
1612