diff options
Diffstat (limited to 'src/google/protobuf/compiler/parser.cc')
-rw-r--r-- | src/google/protobuf/compiler/parser.cc | 400 |
1 files changed, 269 insertions, 131 deletions
diff --git a/src/google/protobuf/compiler/parser.cc b/src/google/protobuf/compiler/parser.cc index 34317b1f..23aa01ce 100644 --- a/src/google/protobuf/compiler/parser.cc +++ b/src/google/protobuf/compiler/parser.cc @@ -174,6 +174,20 @@ bool Parser::ConsumeInteger(int* output, const char* error) { } } +bool Parser::ConsumeSignedInteger(int* output, const char* error) { + bool is_negative = false; + uint64 max_value = kint32max; + if (TryConsume("-")) { + is_negative = true; + max_value += 1; + } + uint64 value = 0; + DO(ConsumeInteger64(max_value, &value, error)); + if (is_negative) value *= -1; + *output = value; + return true; +} + bool Parser::ConsumeInteger64(uint64 max_value, uint64* output, const char* error) { if (LookingAtType(io::Tokenizer::TYPE_INTEGER)) { @@ -237,6 +251,35 @@ bool Parser::ConsumeString(string* output, const char* error) { } } +bool Parser::TryConsumeEndOfDeclaration(const char* text, + const LocationRecorder* location) { + if (LookingAt(text)) { + string leading, trailing; + input_->NextWithComments(&trailing, NULL, &leading); + + // Save the leading comments for next time, and recall the leading comments + // from last time. + leading.swap(upcoming_doc_comments_); + + if (location != NULL) { + location->AttachComments(&leading, &trailing); + } + return true; + } else { + return false; + } +} + +bool Parser::ConsumeEndOfDeclaration(const char* text, + const LocationRecorder* location) { + if (TryConsumeEndOfDeclaration(text, location)) { + return true; + } else { + AddError("Expected \"" + string(text) + "\"."); + return false; + } +} + // ------------------------------------------------------------------- void Parser::AddError(int line, int column, const string& error) { @@ -315,6 +358,19 @@ void Parser::LocationRecorder::RecordLegacyLocation(const Message* descriptor, } } +void Parser::LocationRecorder::AttachComments( + string* leading, string* trailing) const { + GOOGLE_CHECK(!location_->has_leading_comments()); + GOOGLE_CHECK(!location_->has_trailing_comments()); + + if (!leading->empty()) { + location_->mutable_leading_comments()->swap(*leading); + } + if (!trailing->empty()) { + location_->mutable_trailing_comments()->swap(*trailing); + } +} + // ------------------------------------------------------------------- void Parser::SkipStatement() { @@ -322,7 +378,7 @@ void Parser::SkipStatement() { if (AtEnd()) { return; } else if (LookingAtType(io::Tokenizer::TYPE_SYMBOL)) { - if (TryConsume(";")) { + if (TryConsumeEndOfDeclaration(";", NULL)) { return; } else if (TryConsume("{")) { SkipRestOfBlock(); @@ -340,7 +396,7 @@ void Parser::SkipRestOfBlock() { if (AtEnd()) { return; } else if (LookingAtType(io::Tokenizer::TYPE_SYMBOL)) { - if (TryConsume("}")) { + if (TryConsumeEndOfDeclaration("}", NULL)) { return; } else if (TryConsume("{")) { SkipRestOfBlock(); @@ -366,7 +422,7 @@ bool Parser::Parse(io::Tokenizer* input, FileDescriptorProto* file) { if (LookingAtType(io::Tokenizer::TYPE_START)) { // Advance to first token. - input_->Next(); + input_->NextWithComments(NULL, NULL, &upcoming_doc_comments_); } { @@ -393,7 +449,7 @@ bool Parser::Parse(io::Tokenizer* input, FileDescriptorProto* file) { if (LookingAt("}")) { AddError("Unmatched \"}\"."); - input_->Next(); + input_->NextWithComments(NULL, NULL, &upcoming_doc_comments_); } } } @@ -411,7 +467,7 @@ bool Parser::ParseSyntaxIdentifier() { io::Tokenizer::Token syntax_token = input_->current(); string syntax; DO(ConsumeString(&syntax, "Expected syntax identifier.")); - DO(Consume(";")); + DO(ConsumeEndOfDeclaration(";", NULL)); syntax_identifier_ = syntax; @@ -427,7 +483,7 @@ bool Parser::ParseSyntaxIdentifier() { bool Parser::ParseTopLevelStatement(FileDescriptorProto* file, const LocationRecorder& root_location) { - if (TryConsume(";")) { + if (TryConsumeEndOfDeclaration(";", NULL)) { // empty statement; ignore return true; } else if (LookingAt("message")) { @@ -451,14 +507,16 @@ bool Parser::ParseTopLevelStatement(FileDescriptorProto* file, FileDescriptorProto::kMessageTypeFieldNumber, location); } else if (LookingAt("import")) { - int index = file->dependency_size(); - return ParseImport(file->add_dependency(), root_location, index); + return ParseImport(file->mutable_dependency(), + file->mutable_public_dependency(), + file->mutable_weak_dependency(), + root_location); } else if (LookingAt("package")) { return ParsePackage(file, root_location); } else if (LookingAt("option")) { LocationRecorder location(root_location, FileDescriptorProto::kOptionsFieldNumber); - return ParseOption(file->mutable_options(), location); + return ParseOption(file->mutable_options(), location, OPTION_STATEMENT); } else { AddError("Expected top-level statement (e.g. \"message\")."); return false; @@ -482,11 +540,45 @@ bool Parser::ParseMessageDefinition(DescriptorProto* message, return true; } +namespace { + +const int kMaxExtensionRangeSentinel = -1; + +bool IsMessageSetWireFormatMessage(const DescriptorProto& message) { + const MessageOptions& options = message.options(); + for (int i = 0; i < options.uninterpreted_option_size(); ++i) { + const UninterpretedOption& uninterpreted = options.uninterpreted_option(i); + if (uninterpreted.name_size() == 1 && + uninterpreted.name(0).name_part() == "message_set_wire_format" && + uninterpreted.identifier_value() == "true") { + return true; + } + } + return false; +} + +// Modifies any extension ranges that specified 'max' as the end of the +// extension range, and sets them to the type-specific maximum. The actual max +// tag number can only be determined after all options have been parsed. +void AdjustExtensionRangesWithMaxEndNumber(DescriptorProto* message) { + const bool is_message_set = IsMessageSetWireFormatMessage(*message); + const int max_extension_number = is_message_set ? + kint32max : + FieldDescriptor::kMaxNumber + 1; + for (int i = 0; i < message->extension_range_size(); ++i) { + if (message->extension_range(i).end() == kMaxExtensionRangeSentinel) { + message->mutable_extension_range(i)->set_end(max_extension_number); + } + } +} + +} // namespace + bool Parser::ParseMessageBlock(DescriptorProto* message, const LocationRecorder& message_location) { - DO(Consume("{")); + DO(ConsumeEndOfDeclaration("{", &message_location)); - while (!TryConsume("}")) { + while (!TryConsumeEndOfDeclaration("}", NULL)) { if (AtEnd()) { AddError("Reached end of input in message definition (missing '}')."); return false; @@ -499,12 +591,15 @@ bool Parser::ParseMessageBlock(DescriptorProto* message, } } + if (message->extension_range_size() > 0) { + AdjustExtensionRangesWithMaxEndNumber(message); + } return true; } bool Parser::ParseMessageStatement(DescriptorProto* message, const LocationRecorder& message_location) { - if (TryConsume(";")) { + if (TryConsumeEndOfDeclaration(";", NULL)) { // empty statement; ignore return true; } else if (LookingAt("message")) { @@ -532,7 +627,7 @@ bool Parser::ParseMessageStatement(DescriptorProto* message, } else if (LookingAt("option")) { LocationRecorder location(message_location, DescriptorProto::kOptionsFieldNumber); - return ParseOption(message->mutable_options(), location); + return ParseOption(message->mutable_options(), location, OPTION_STATEMENT); } else { LocationRecorder location(message_location, DescriptorProto::kFieldFieldNumber, @@ -647,7 +742,7 @@ bool Parser::ParseMessageField(FieldDescriptorProto* field, return false; } } else { - DO(Consume(";")); + DO(ConsumeEndOfDeclaration(";", &field_location)); } return true; @@ -669,7 +764,7 @@ bool Parser::ParseFieldOptions(FieldDescriptorProto* field, // the default value is not actually an option. DO(ParseDefaultAssignment(field, field_location)); } else { - DO(ParseOptionAssignment(field->mutable_options(), location)); + DO(ParseOption(field->mutable_options(), location, OPTION_ASSIGNMENT)); } } while (TryConsume(",")); @@ -835,6 +930,8 @@ bool Parser::ParseOptionNamePart(UninterpretedOption* uninterpreted_option, bool Parser::ParseUninterpretedBlock(string* value) { // Note that enclosing braces are not added to *value. + // We do NOT use ConsumeEndOfStatement for this brace because it's delimiting + // an expression, not a block of statements. DO(Consume("{")); int brace_depth = 1; while (!AtEnd()) { @@ -858,8 +955,9 @@ bool Parser::ParseUninterpretedBlock(string* value) { // We don't interpret the option here. Instead we store it in an // UninterpretedOption, to be interpreted later. -bool Parser::ParseOptionAssignment(Message* options, - const LocationRecorder& options_location) { +bool Parser::ParseOption(Message* options, + const LocationRecorder& options_location, + OptionStyle style) { // Create an entry in the uninterpreted_option field. const FieldDescriptor* uninterpreted_option_field = options->GetDescriptor()-> FindFieldByName("uninterpreted_option"); @@ -872,6 +970,10 @@ bool Parser::ParseOptionAssignment(Message* options, options_location, uninterpreted_option_field->number(), reflection->FieldSize(*options, uninterpreted_option_field)); + if (style == OPTION_STATEMENT) { + DO(Consume("option")); + } + UninterpretedOption* uninterpreted_option = down_cast<UninterpretedOption*>( options->GetReflection()->AddMessage(options, uninterpreted_option_field)); @@ -899,82 +1001,91 @@ bool Parser::ParseOptionAssignment(Message* options, DO(Consume("=")); - LocationRecorder value_location(location); - value_location.RecordLegacyLocation( - uninterpreted_option, DescriptorPool::ErrorCollector::OPTION_VALUE); - - // All values are a single token, except for negative numbers, which consist - // of a single '-' symbol, followed by a positive number. - bool is_negative = TryConsume("-"); + { + LocationRecorder value_location(location); + value_location.RecordLegacyLocation( + uninterpreted_option, DescriptorPool::ErrorCollector::OPTION_VALUE); - switch (input_->current().type) { - case io::Tokenizer::TYPE_START: - GOOGLE_LOG(FATAL) << "Trying to read value before any tokens have been read."; - return false; + // All values are a single token, except for negative numbers, which consist + // of a single '-' symbol, followed by a positive number. + bool is_negative = TryConsume("-"); - case io::Tokenizer::TYPE_END: - AddError("Unexpected end of stream while parsing option value."); - return false; + switch (input_->current().type) { + case io::Tokenizer::TYPE_START: + GOOGLE_LOG(FATAL) << "Trying to read value before any tokens have been read."; + return false; - case io::Tokenizer::TYPE_IDENTIFIER: { - value_location.AddPath(UninterpretedOption::kIdentifierValueFieldNumber); - if (is_negative) { - AddError("Invalid '-' symbol before identifier."); + case io::Tokenizer::TYPE_END: + AddError("Unexpected end of stream while parsing option value."); return false; - } - string value; - DO(ConsumeIdentifier(&value, "Expected identifier.")); - uninterpreted_option->set_identifier_value(value); - break; - } - case io::Tokenizer::TYPE_INTEGER: { - uint64 value; - uint64 max_value = - is_negative ? static_cast<uint64>(kint64max) + 1 : kuint64max; - DO(ConsumeInteger64(max_value, &value, "Expected integer.")); - if (is_negative) { + case io::Tokenizer::TYPE_IDENTIFIER: { value_location.AddPath( - UninterpretedOption::kNegativeIntValueFieldNumber); - uninterpreted_option->set_negative_int_value(-static_cast<int64>(value)); - } else { - value_location.AddPath( - UninterpretedOption::kPositiveIntValueFieldNumber); - uninterpreted_option->set_positive_int_value(value); + UninterpretedOption::kIdentifierValueFieldNumber); + if (is_negative) { + AddError("Invalid '-' symbol before identifier."); + return false; + } + string value; + DO(ConsumeIdentifier(&value, "Expected identifier.")); + uninterpreted_option->set_identifier_value(value); + break; } - break; - } - case io::Tokenizer::TYPE_FLOAT: { - value_location.AddPath(UninterpretedOption::kDoubleValueFieldNumber); - double value; - DO(ConsumeNumber(&value, "Expected number.")); - uninterpreted_option->set_double_value(is_negative ? -value : value); - break; - } + case io::Tokenizer::TYPE_INTEGER: { + uint64 value; + uint64 max_value = + is_negative ? static_cast<uint64>(kint64max) + 1 : kuint64max; + DO(ConsumeInteger64(max_value, &value, "Expected integer.")); + if (is_negative) { + value_location.AddPath( + UninterpretedOption::kNegativeIntValueFieldNumber); + uninterpreted_option->set_negative_int_value( + -static_cast<int64>(value)); + } else { + value_location.AddPath( + UninterpretedOption::kPositiveIntValueFieldNumber); + uninterpreted_option->set_positive_int_value(value); + } + break; + } - case io::Tokenizer::TYPE_STRING: { - value_location.AddPath(UninterpretedOption::kStringValueFieldNumber); - if (is_negative) { - AddError("Invalid '-' symbol before string."); - return false; + case io::Tokenizer::TYPE_FLOAT: { + value_location.AddPath(UninterpretedOption::kDoubleValueFieldNumber); + double value; + DO(ConsumeNumber(&value, "Expected number.")); + uninterpreted_option->set_double_value(is_negative ? -value : value); + break; } - string value; - DO(ConsumeString(&value, "Expected string.")); - uninterpreted_option->set_string_value(value); - break; - } - case io::Tokenizer::TYPE_SYMBOL: - if (LookingAt("{")) { - value_location.AddPath(UninterpretedOption::kAggregateValueFieldNumber); - DO(ParseUninterpretedBlock( - uninterpreted_option->mutable_aggregate_value())); - } else { - AddError("Expected option value."); - return false; + case io::Tokenizer::TYPE_STRING: { + value_location.AddPath(UninterpretedOption::kStringValueFieldNumber); + if (is_negative) { + AddError("Invalid '-' symbol before string."); + return false; + } + string value; + DO(ConsumeString(&value, "Expected string.")); + uninterpreted_option->set_string_value(value); + break; } - break; + + case io::Tokenizer::TYPE_SYMBOL: + if (LookingAt("{")) { + value_location.AddPath( + UninterpretedOption::kAggregateValueFieldNumber); + DO(ParseUninterpretedBlock( + uninterpreted_option->mutable_aggregate_value())); + } else { + AddError("Expected option value."); + return false; + } + break; + } + } + + if (style == OPTION_STATEMENT) { + DO(ConsumeEndOfDeclaration(";", &location)); } return true; @@ -1008,7 +1119,10 @@ bool Parser::ParseExtensions(DescriptorProto* message, LocationRecorder end_location( location, DescriptorProto::ExtensionRange::kEndFieldNumber); if (TryConsume("max")) { - end = FieldDescriptor::kMaxNumber; + // Set to the sentinel value - 1 since we increment the value below. + // The actual value of the end of the range should be set with + // AdjustExtensionRangesWithMaxEndNumber. + end = kMaxExtensionRangeSentinel - 1; } else { DO(ConsumeInteger(&end, "Expected integer.")); } @@ -1028,7 +1142,7 @@ bool Parser::ParseExtensions(DescriptorProto* message, range->set_end(end); } while (TryConsume(",")); - DO(Consume(";")); + DO(ConsumeEndOfDeclaration(";", &extensions_location)); return true; } @@ -1046,7 +1160,7 @@ bool Parser::ParseExtend(RepeatedPtrField<FieldDescriptorProto>* extensions, io::Tokenizer::Token extendee_end = input_->previous(); // Parse the block. - DO(Consume("{")); + DO(ConsumeEndOfDeclaration("{", &extend_location)); bool is_first = true; @@ -1083,7 +1197,7 @@ bool Parser::ParseExtend(RepeatedPtrField<FieldDescriptorProto>* extensions, // other statements. SkipStatement(); } - } while(!TryConsume("}")); + } while (!TryConsumeEndOfDeclaration("}", NULL)); return true; } @@ -1109,9 +1223,9 @@ bool Parser::ParseEnumDefinition(EnumDescriptorProto* enum_type, bool Parser::ParseEnumBlock(EnumDescriptorProto* enum_type, const LocationRecorder& enum_location) { - DO(Consume("{")); + DO(ConsumeEndOfDeclaration("{", &enum_location)); - while (!TryConsume("}")) { + while (!TryConsumeEndOfDeclaration("}", NULL)) { if (AtEnd()) { AddError("Reached end of input in enum definition (missing '}')."); return false; @@ -1129,13 +1243,14 @@ bool Parser::ParseEnumBlock(EnumDescriptorProto* enum_type, bool Parser::ParseEnumStatement(EnumDescriptorProto* enum_type, const LocationRecorder& enum_location) { - if (TryConsume(";")) { + if (TryConsumeEndOfDeclaration(";", NULL)) { // empty statement; ignore return true; } else if (LookingAt("option")) { LocationRecorder location(enum_location, EnumDescriptorProto::kOptionsFieldNumber); - return ParseOption(enum_type->mutable_options(), location); + return ParseOption(enum_type->mutable_options(), location, + OPTION_STATEMENT); } else { LocationRecorder location(enum_location, EnumDescriptorProto::kValueFieldNumber, enum_type->value_size()); @@ -1164,16 +1279,14 @@ bool Parser::ParseEnumConstant(EnumValueDescriptorProto* enum_value, location.RecordLegacyLocation( enum_value, DescriptorPool::ErrorCollector::NUMBER); - bool is_negative = TryConsume("-"); int number; - DO(ConsumeInteger(&number, "Expected integer.")); - if (is_negative) number *= -1; + DO(ConsumeSignedInteger(&number, "Expected integer.")); enum_value->set_number(number); } DO(ParseEnumConstantOptions(enum_value, enum_value_location)); - DO(Consume(";")); + DO(ConsumeEndOfDeclaration(";", &enum_value_location)); return true; } @@ -1189,7 +1302,7 @@ bool Parser::ParseEnumConstantOptions( DO(Consume("[")); do { - DO(ParseOptionAssignment(value->mutable_options(), location)); + DO(ParseOption(value->mutable_options(), location, OPTION_ASSIGNMENT)); } while (TryConsume(",")); DO(Consume("]")); @@ -1217,9 +1330,9 @@ bool Parser::ParseServiceDefinition(ServiceDescriptorProto* service, bool Parser::ParseServiceBlock(ServiceDescriptorProto* service, const LocationRecorder& service_location) { - DO(Consume("{")); + DO(ConsumeEndOfDeclaration("{", &service_location)); - while (!TryConsume("}")) { + while (!TryConsumeEndOfDeclaration("}", NULL)) { if (AtEnd()) { AddError("Reached end of input in service definition (missing '}')."); return false; @@ -1237,13 +1350,13 @@ bool Parser::ParseServiceBlock(ServiceDescriptorProto* service, bool Parser::ParseServiceStatement(ServiceDescriptorProto* service, const LocationRecorder& service_location) { - if (TryConsume(";")) { + if (TryConsumeEndOfDeclaration(";", NULL)) { // empty statement; ignore return true; } else if (LookingAt("option")) { LocationRecorder location( service_location, ServiceDescriptorProto::kOptionsFieldNumber); - return ParseOption(service->mutable_options(), location); + return ParseOption(service->mutable_options(), location, OPTION_STATEMENT); } else { LocationRecorder location(service_location, ServiceDescriptorProto::kMethodFieldNumber, service->method_size()); @@ -1286,28 +1399,41 @@ bool Parser::ParseServiceMethod(MethodDescriptorProto* method, } DO(Consume(")")); - if (TryConsume("{")) { + if (LookingAt("{")) { // Options! - while (!TryConsume("}")) { - if (AtEnd()) { - AddError("Reached end of input in method options (missing '}')."); - return false; - } + DO(ParseOptions(method_location, + MethodDescriptorProto::kOptionsFieldNumber, + method->mutable_options())); + } else { + DO(ConsumeEndOfDeclaration(";", &method_location)); + } - if (TryConsume(";")) { - // empty statement; ignore - } else { - LocationRecorder location(method_location, - MethodDescriptorProto::kOptionsFieldNumber); - if (!ParseOption(method->mutable_options(), location)) { - // This statement failed to parse. Skip it, but keep looping to - // parse other statements. - SkipStatement(); - } + return true; +} + + +bool Parser::ParseOptions(const LocationRecorder& parent_location, + const int optionsFieldNumber, + Message* mutable_options) { + // Options! + ConsumeEndOfDeclaration("{", &parent_location); + while (!TryConsumeEndOfDeclaration("}", NULL)) { + if (AtEnd()) { + AddError("Reached end of input in method options (missing '}')."); + return false; + } + + if (TryConsumeEndOfDeclaration(";", NULL)) { + // empty statement; ignore + } else { + LocationRecorder location(parent_location, + optionsFieldNumber); + if (!ParseOption(mutable_options, location, OPTION_STATEMENT)) { + // This statement failed to parse. Skip it, but keep looping to + // parse other statements. + SkipStatement(); } } - } else { - DO(Consume(";")); } return true; @@ -1406,32 +1532,44 @@ bool Parser::ParsePackage(FileDescriptorProto* file, if (!TryConsume(".")) break; file->mutable_package()->append("."); } + + location.EndAt(input_->previous()); + + DO(ConsumeEndOfDeclaration(";", &location)); } - DO(Consume(";")); return true; } -bool Parser::ParseImport(string* import_filename, - const LocationRecorder& root_location, - int index) { +bool Parser::ParseImport(RepeatedPtrField<string>* dependency, + RepeatedField<int32>* public_dependency, + RepeatedField<int32>* weak_dependency, + const LocationRecorder& root_location) { DO(Consume("import")); + if (LookingAt("public")) { + LocationRecorder location( + root_location, FileDescriptorProto::kPublicDependencyFieldNumber, + public_dependency->size()); + DO(Consume("public")); + *public_dependency->Add() = dependency->size(); + } else if (LookingAt("weak")) { + LocationRecorder location( + root_location, FileDescriptorProto::kWeakDependencyFieldNumber, + weak_dependency->size()); + DO(Consume("weak")); + *weak_dependency->Add() = dependency->size(); + } { LocationRecorder location(root_location, FileDescriptorProto::kDependencyFieldNumber, - index); - DO(ConsumeString(import_filename, + dependency->size()); + DO(ConsumeString(dependency->Add(), "Expected a string naming the file to import.")); - } - DO(Consume(";")); - return true; -} -bool Parser::ParseOption(Message* options, - const LocationRecorder& options_location) { - DO(Consume("option")); - DO(ParseOptionAssignment(options, options_location)); - DO(Consume(";")); + location.EndAt(input_->previous()); + + DO(ConsumeEndOfDeclaration(";", &location)); + } return true; } |