diff options
Diffstat (limited to 'src/google/protobuf/descriptor.cc')
-rw-r--r-- | src/google/protobuf/descriptor.cc | 660 |
1 files changed, 549 insertions, 111 deletions
diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index 754b5628..a60abf43 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -54,7 +54,7 @@ #include <google/protobuf/stubs/strutil.h> #include <google/protobuf/stubs/substitute.h> #include <google/protobuf/stubs/map-util.h> -#include <google/protobuf/stubs/stl_util-inl.h> +#include <google/protobuf/stubs/stl_util.h> #undef PACKAGE // autoheader #defines this. :( @@ -108,6 +108,21 @@ const char * const FieldDescriptor::kTypeToName[MAX_TYPE + 1] = { "sint64", // TYPE_SINT64 }; +const char * const FieldDescriptor::kCppTypeToName[MAX_CPPTYPE + 1] = { + "ERROR", // 0 is reserved for errors + + "int32", // CPPTYPE_INT32 + "int64", // CPPTYPE_INT64 + "uint32", // CPPTYPE_UINT32 + "uint64", // CPPTYPE_UINT64 + "double", // CPPTYPE_DOUBLE + "float", // CPPTYPE_FLOAT + "bool", // CPPTYPE_BOOL + "enum", // CPPTYPE_ENUM + "string", // CPPTYPE_STRING + "message", // CPPTYPE_MESSAGE +}; + const char * const FieldDescriptor::kLabelToName[MAX_LABEL + 1] = { "ERROR", // 0 is reserved for errors @@ -221,7 +236,8 @@ struct PointerStringPairHash { struct Symbol { enum Type { - NULL_SYMBOL, MESSAGE, FIELD, ENUM, ENUM_VALUE, SERVICE, METHOD, PACKAGE + NULL_SYMBOL, MESSAGE, FIELD, ENUM, ENUM_VALUE, SERVICE, METHOD, + PACKAGE }; Type type; union { @@ -311,17 +327,42 @@ class DescriptorPool::Tables { Tables(); ~Tables(); - // Checkpoint the state of the tables. Future calls to Rollback() will - // return the Tables to this state. This is used when building files, since - // some kinds of validation errors cannot be detected until the file's - // descriptors have already been added to the tables. BuildFile() calls - // Checkpoint() before it starts building and Rollback() if it encounters - // an error. - void Checkpoint(); + // Record the current state of the tables to the stack of checkpoints. + // Each call to AddCheckpoint() must be paired with exactly one call to either + // ClearLastCheckpoint() or RollbackToLastCheckpoint(). + // + // This is used when building files, since some kinds of validation errors + // cannot be detected until the file's descriptors have already been added to + // the tables. + // + // This supports recursive checkpoints, since building a file may trigger + // recursive building of other files. Note that recursive checkpoints are not + // normally necessary; explicit dependencies are built prior to checkpointing. + // So although we recursively build transitive imports, there is at most one + // checkpoint in the stack during dependency building. + // + // Recursive checkpoints only arise during cross-linking of the descriptors. + // Symbol references must be resolved, via DescriptorBuilder::FindSymbol and + // friends. If the pending file references an unknown symbol + // (e.g., it is not defined in the pending file's explicit dependencies), and + // the pool is using a fallback database, and that database contains a file + // defining that symbol, and that file has not yet been built by the pool, + // the pool builds the file during cross-linking, leading to another + // checkpoint. + void AddCheckpoint(); + + // Mark the last checkpoint as having cleared successfully, removing it from + // the stack. If the stack is empty, all pending symbols will be committed. + // + // Note that this does not guarantee that the symbols added since the last + // checkpoint won't be rolled back: if a checkpoint gets rolled back, + // everything past that point gets rolled back, including symbols added after + // checkpoints that were pushed onto the stack after it and marked as cleared. + void ClearLastCheckpoint(); - // Roll back the Tables to the state of the last Checkpoint(), removing - // everything that was added after that point. - void Rollback(); + // Roll back the Tables to the state of the checkpoint at the top of the + // stack, removing everything that was added after that point. + void RollbackToLastCheckpoint(); // The stack of files which are currently being built. Used to detect // cyclic dependencies when loading files from a DescriptorDatabase. Not @@ -405,10 +446,28 @@ class DescriptorPool::Tables { FilesByNameMap files_by_name_; ExtensionsGroupedByDescriptorMap extensions_; - int strings_before_checkpoint_; - int messages_before_checkpoint_; - int file_tables_before_checkpoint_; - int allocations_before_checkpoint_; + struct CheckPoint { + explicit CheckPoint(const Tables* tables) + : strings_before_checkpoint(tables->strings_.size()), + messages_before_checkpoint(tables->messages_.size()), + file_tables_before_checkpoint(tables->file_tables_.size()), + allocations_before_checkpoint(tables->allocations_.size()), + pending_symbols_before_checkpoint( + tables->symbols_after_checkpoint_.size()), + pending_files_before_checkpoint( + tables->files_after_checkpoint_.size()), + pending_extensions_before_checkpoint( + tables->extensions_after_checkpoint_.size()) { + } + int strings_before_checkpoint; + int messages_before_checkpoint; + int file_tables_before_checkpoint; + int allocations_before_checkpoint; + int pending_symbols_before_checkpoint; + int pending_files_before_checkpoint; + int pending_extensions_before_checkpoint; + }; + vector<CheckPoint> checkpoints_; vector<const char* > symbols_after_checkpoint_; vector<const char* > files_after_checkpoint_; vector<DescriptorIntPair> extensions_after_checkpoint_; @@ -481,11 +540,15 @@ class FileDescriptorTables { }; DescriptorPool::Tables::Tables() - : strings_before_checkpoint_(0), - messages_before_checkpoint_(0), - allocations_before_checkpoint_(0) {} + // Start some hash_map and hash_set objects with a small # of buckets + : known_bad_files_(3), + extensions_loaded_from_db_(3), + symbols_by_name_(3), + files_by_name_(3) {} + DescriptorPool::Tables::~Tables() { + GOOGLE_DCHECK(checkpoints_.empty()); // Note that the deletion order is important, since the destructors of some // messages may refer to objects in allocations_. STLDeleteElements(&messages_); @@ -496,51 +559,80 @@ DescriptorPool::Tables::~Tables() { STLDeleteElements(&file_tables_); } -FileDescriptorTables::FileDescriptorTables() {} +FileDescriptorTables::FileDescriptorTables() + // Initialize all the hash tables to start out with a small # of buckets + : symbols_by_parent_(3), + fields_by_lowercase_name_(3), + fields_by_camelcase_name_(3), + fields_by_number_(3), + enum_values_by_number_(3) { +} + FileDescriptorTables::~FileDescriptorTables() {} const FileDescriptorTables FileDescriptorTables::kEmpty; -void DescriptorPool::Tables::Checkpoint() { - strings_before_checkpoint_ = strings_.size(); - messages_before_checkpoint_ = messages_.size(); - file_tables_before_checkpoint_ = file_tables_.size(); - allocations_before_checkpoint_ = allocations_.size(); +void DescriptorPool::Tables::AddCheckpoint() { + checkpoints_.push_back(CheckPoint(this)); +} - symbols_after_checkpoint_.clear(); - files_after_checkpoint_.clear(); - extensions_after_checkpoint_.clear(); +void DescriptorPool::Tables::ClearLastCheckpoint() { + GOOGLE_DCHECK(!checkpoints_.empty()); + checkpoints_.pop_back(); + if (checkpoints_.empty()) { + // All checkpoints have been cleared: we can now commit all of the pending + // data. + symbols_after_checkpoint_.clear(); + files_after_checkpoint_.clear(); + extensions_after_checkpoint_.clear(); + } } -void DescriptorPool::Tables::Rollback() { - for (int i = 0; i < symbols_after_checkpoint_.size(); i++) { +void DescriptorPool::Tables::RollbackToLastCheckpoint() { + GOOGLE_DCHECK(!checkpoints_.empty()); + const CheckPoint& checkpoint = checkpoints_.back(); + + for (int i = checkpoint.pending_symbols_before_checkpoint; + i < symbols_after_checkpoint_.size(); + i++) { symbols_by_name_.erase(symbols_after_checkpoint_[i]); } - for (int i = 0; i < files_after_checkpoint_.size(); i++) { + for (int i = checkpoint.pending_files_before_checkpoint; + i < files_after_checkpoint_.size(); + i++) { files_by_name_.erase(files_after_checkpoint_[i]); } - for (int i = 0; i < extensions_after_checkpoint_.size(); i++) { + for (int i = checkpoint.pending_extensions_before_checkpoint; + i < extensions_after_checkpoint_.size(); + i++) { extensions_.erase(extensions_after_checkpoint_[i]); } - symbols_after_checkpoint_.clear(); - files_after_checkpoint_.clear(); - extensions_after_checkpoint_.clear(); + symbols_after_checkpoint_.resize( + checkpoint.pending_symbols_before_checkpoint); + files_after_checkpoint_.resize(checkpoint.pending_files_before_checkpoint); + extensions_after_checkpoint_.resize( + checkpoint.pending_extensions_before_checkpoint); STLDeleteContainerPointers( - strings_.begin() + strings_before_checkpoint_, strings_.end()); + strings_.begin() + checkpoint.strings_before_checkpoint, strings_.end()); STLDeleteContainerPointers( - messages_.begin() + messages_before_checkpoint_, messages_.end()); + messages_.begin() + checkpoint.messages_before_checkpoint, + messages_.end()); STLDeleteContainerPointers( - file_tables_.begin() + file_tables_before_checkpoint_, file_tables_.end()); - for (int i = allocations_before_checkpoint_; i < allocations_.size(); i++) { + file_tables_.begin() + checkpoint.file_tables_before_checkpoint, + file_tables_.end()); + for (int i = checkpoint.allocations_before_checkpoint; + i < allocations_.size(); + i++) { operator delete(allocations_[i]); } - strings_.resize(strings_before_checkpoint_); - messages_.resize(messages_before_checkpoint_); - file_tables_.resize(file_tables_before_checkpoint_); - allocations_.resize(allocations_before_checkpoint_); + strings_.resize(checkpoint.strings_before_checkpoint); + messages_.resize(checkpoint.messages_before_checkpoint); + file_tables_.resize(checkpoint.file_tables_before_checkpoint); + allocations_.resize(checkpoint.allocations_before_checkpoint); + checkpoints_.pop_back(); } // ------------------------------------------------------------------- @@ -811,7 +903,7 @@ void DeleteGeneratedPool() { generated_pool_ = NULL; } -void InitGeneratedPool() { +static void InitGeneratedPool() { generated_database_ = new EncodedDescriptorDatabase; generated_pool_ = new DescriptorPool(generated_database_); @@ -874,11 +966,11 @@ const FileDescriptor* DescriptorPool::FindFileByName(const string& name) const { const FileDescriptor* result = tables_->FindFile(name); if (result != NULL) return result; if (underlay_ != NULL) { - const FileDescriptor* result = underlay_->FindFileByName(name); + result = underlay_->FindFileByName(name); if (result != NULL) return result; } if (TryFindFileInFallbackDatabase(name)) { - const FileDescriptor* result = tables_->FindFile(name); + result = tables_->FindFile(name); if (result != NULL) return result; } return NULL; @@ -890,12 +982,12 @@ const FileDescriptor* DescriptorPool::FindFileContainingSymbol( Symbol result = tables_->FindSymbol(symbol_name); if (!result.IsNull()) return result.GetFile(); if (underlay_ != NULL) { - const FileDescriptor* result = + const FileDescriptor* file_result = underlay_->FindFileContainingSymbol(symbol_name); - if (result != NULL) return result; + if (file_result != NULL) return file_result; } if (TryFindSymbolInFallbackDatabase(symbol_name)) { - Symbol result = tables_->FindSymbol(symbol_name); + result = tables_->FindSymbol(symbol_name); if (!result.IsNull()) return result.GetFile(); } return NULL; @@ -962,12 +1054,11 @@ const FieldDescriptor* DescriptorPool::FindExtensionByNumber( return result; } if (underlay_ != NULL) { - const FieldDescriptor* result = - underlay_->FindExtensionByNumber(extendee, number); + result = underlay_->FindExtensionByNumber(extendee, number); if (result != NULL) return result; } if (TryFindExtensionInFallbackDatabase(extendee, number)) { - const FieldDescriptor* result = tables_->FindExtension(extendee, number); + result = tables_->FindExtension(extendee, number); if (result != NULL) { return result; } @@ -1241,9 +1332,48 @@ bool DescriptorPool::TryFindFileInFallbackDatabase(const string& name) const { return true; } +bool DescriptorPool::IsSubSymbolOfBuiltType(const string& name) const { + string prefix = name; + for (;;) { + string::size_type dot_pos = prefix.find_last_of('.'); + if (dot_pos == string::npos) { + break; + } + prefix = prefix.substr(0, dot_pos); + Symbol symbol = tables_->FindSymbol(prefix); + // If the symbol type is anything other than PACKAGE, then its complete + // definition is already known. + if (!symbol.IsNull() && symbol.type != Symbol::PACKAGE) { + return true; + } + } + if (underlay_ != NULL) { + // Check to see if any prefix of this symbol exists in the underlay. + return underlay_->IsSubSymbolOfBuiltType(name); + } + return false; +} + bool DescriptorPool::TryFindSymbolInFallbackDatabase(const string& name) const { if (fallback_database_ == NULL) return false; + // We skip looking in the fallback database if the name is a sub-symbol of + // any descriptor that already exists in the descriptor pool (except for + // package descriptors). This is valid because all symbols except for + // packages are defined in a single file, so if the symbol exists then we + // should already have its definition. + // + // The other reason to do this is to support "overriding" type definitions + // by merging two databases that define the same type. (Yes, people do + // this.) The main difficulty with making this work is that + // FindFileContainingSymbol() is allowed to return both false positives + // (e.g., SimpleDescriptorDatabase, UpgradedDescriptorDatabase) and false + // negatives (e.g. ProtoFileParser, SourceTreeDescriptorDatabase). When two + // such databases are merged, looking up a non-existent sub-symbol of a type + // that already exists in the descriptor pool can result in an attempt to + // load multiple definitions of the same type. The check below avoids this. + if (IsSubSymbolOfBuiltType(name)) return false; + FileDescriptorProto file_proto; if (!fallback_database_->FindFileContainingSymbol(name, &file_proto)) { return false; @@ -1345,6 +1475,14 @@ void FileDescriptor::CopyTo(FileDescriptorProto* proto) const { proto->add_dependency(dependency(i)->name()); } + for (int i = 0; i < public_dependency_count(); i++) { + proto->add_public_dependency(public_dependencies_[i]); + } + + for (int i = 0; i < weak_dependency_count(); i++) { + proto->add_weak_dependency(weak_dependencies_[i]); + } + for (int i = 0; i < message_type_count(); i++) { message_type(i)->CopyTo(proto->add_message_type()); } @@ -1363,6 +1501,12 @@ void FileDescriptor::CopyTo(FileDescriptorProto* proto) const { } } +void FileDescriptor::CopySourceCodeInfoTo(FileDescriptorProto* proto) const { + if (source_code_info_ != &SourceCodeInfo::default_instance()) { + proto->mutable_source_code_info()->CopyFrom(*source_code_info_); + } +} + void Descriptor::CopyTo(DescriptorProto* proto) const { proto->set_name(name()); @@ -1490,16 +1634,14 @@ void MethodDescriptor::CopyTo(MethodDescriptorProto* proto) const { namespace { // Used by each of the option formatters. -bool RetrieveOptions(const Message &options, vector<string> *option_entries) { +bool RetrieveOptions(int depth, + const Message &options, + vector<string> *option_entries) { option_entries->clear(); const Reflection* reflection = options.GetReflection(); vector<const FieldDescriptor*> fields; reflection->ListFields(options, &fields); for (int i = 0; i < fields.size(); i++) { - // Doesn't make sense to have message type fields here - if (fields[i]->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { - continue; - } int count = 1; bool repeated = false; if (fields[i]->is_repeated()) { @@ -1508,9 +1650,27 @@ bool RetrieveOptions(const Message &options, vector<string> *option_entries) { } for (int j = 0; j < count; j++) { string fieldval; - TextFormat::PrintFieldValueToString(options, fields[i], - repeated ? count : -1, &fieldval); - option_entries->push_back(fields[i]->name() + " = " + fieldval); + if (fields[i]->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + string tmp; + TextFormat::Printer printer; + printer.SetInitialIndentLevel(depth + 1); + printer.PrintFieldValueToString(options, fields[i], + repeated ? j : -1, &tmp); + fieldval.append("{\n"); + fieldval.append(tmp); + fieldval.append(depth * 2, ' '); + fieldval.append("}"); + } else { + TextFormat::PrintFieldValueToString(options, fields[i], + repeated ? j : -1, &fieldval); + } + string name; + if (fields[i]->is_extension()) { + name = "(." + fields[i]->full_name() + ")"; + } else { + name = fields[i]->name(); + } + option_entries->push_back(name + " = " + fieldval); } } return !option_entries->empty(); @@ -1518,9 +1678,9 @@ bool RetrieveOptions(const Message &options, vector<string> *option_entries) { // Formats options that all appear together in brackets. Does not include // brackets. -bool FormatBracketedOptions(const Message &options, string *output) { +bool FormatBracketedOptions(int depth, const Message &options, string *output) { vector<string> all_options; - if (RetrieveOptions(options, &all_options)) { + if (RetrieveOptions(depth, options, &all_options)) { output->append(JoinStrings(all_options, ", ")); } return !all_options.empty(); @@ -1530,7 +1690,7 @@ bool FormatBracketedOptions(const Message &options, string *output) { bool FormatLineOptions(int depth, const Message &options, string *output) { string prefix(depth * 2, ' '); vector<string> all_options; - if (RetrieveOptions(options, &all_options)) { + if (RetrieveOptions(depth, options, &all_options)) { for (int i = 0; i < all_options.size(); i++) { strings::SubstituteAndAppend(output, "$0option $1;\n", prefix, all_options[i]); @@ -1544,9 +1704,24 @@ bool FormatLineOptions(int depth, const Message &options, string *output) { string FileDescriptor::DebugString() const { string contents = "syntax = \"proto2\";\n\n"; + set<int> public_dependencies; + set<int> weak_dependencies; + public_dependencies.insert(public_dependencies_, + public_dependencies_ + public_dependency_count_); + weak_dependencies.insert(weak_dependencies_, + weak_dependencies_ + weak_dependency_count_); + for (int i = 0; i < dependency_count(); i++) { - strings::SubstituteAndAppend(&contents, "import \"$0\";\n", - dependency(i)->name()); + if (public_dependencies.count(i) > 0) { + strings::SubstituteAndAppend(&contents, "import public \"$0\";\n", + dependency(i)->name()); + } else if (weak_dependencies.count(i) > 0) { + strings::SubstituteAndAppend(&contents, "import weak \"$0\";\n", + dependency(i)->name()); + } else { + strings::SubstituteAndAppend(&contents, "import \"$0\";\n", + dependency(i)->name()); + } } if (!package().empty()) { @@ -1712,7 +1887,7 @@ void FieldDescriptor::DebugString(int depth, string *contents) const { } string formatted_options; - if (FormatBracketedOptions(options(), &formatted_options)) { + if (FormatBracketedOptions(depth, options(), &formatted_options)) { contents->append(bracketed ? ", " : " ["); bracketed = true; contents->append(formatted_options); @@ -1761,7 +1936,7 @@ void EnumValueDescriptor::DebugString(int depth, string *contents) const { prefix, name(), number()); string formatted_options; - if (FormatBracketedOptions(options(), &formatted_options)) { + if (FormatBracketedOptions(depth, options(), &formatted_options)) { strings::SubstituteAndAppend(contents, " [$0]", formatted_options); } contents->append(";\n"); @@ -1807,6 +1982,126 @@ void MethodDescriptor::DebugString(int depth, string *contents) const { contents->append(";\n"); } } + + +// Location methods =============================================== + +static bool PathsEqual(const vector<int>& x, const RepeatedField<int32>& y) { + if (x.size() != y.size()) return false; + for (int i = 0; i < x.size(); ++i) { + if (x[i] != y.Get(i)) return false; + } + return true; +} + +bool FileDescriptor::GetSourceLocation(const vector<int>& path, + SourceLocation* out_location) const { + GOOGLE_CHECK_NOTNULL(out_location); + const SourceCodeInfo* info = source_code_info_; + for (int i = 0; info && i < info->location_size(); ++i) { + if (PathsEqual(path, info->location(i).path())) { + const RepeatedField<int32>& span = info->location(i).span(); + if (span.size() == 3 || span.size() == 4) { + out_location->start_line = span.Get(0); + out_location->start_column = span.Get(1); + out_location->end_line = span.Get(span.size() == 3 ? 0 : 2); + out_location->end_column = span.Get(span.size() - 1); + + out_location->leading_comments = info->location(i).leading_comments(); + out_location->trailing_comments = info->location(i).trailing_comments(); + return true; + } + } + } + return false; +} + +bool FieldDescriptor::is_packed() const { + return is_packable() && (options_ != NULL) && options_->packed(); +} + +bool Descriptor::GetSourceLocation(SourceLocation* out_location) const { + vector<int> path; + GetLocationPath(&path); + return file()->GetSourceLocation(path, out_location); +} + +bool FieldDescriptor::GetSourceLocation(SourceLocation* out_location) const { + vector<int> path; + GetLocationPath(&path); + return file()->GetSourceLocation(path, out_location); +} + +bool EnumDescriptor::GetSourceLocation(SourceLocation* out_location) const { + vector<int> path; + GetLocationPath(&path); + return file()->GetSourceLocation(path, out_location); +} + +bool MethodDescriptor::GetSourceLocation(SourceLocation* out_location) const { + vector<int> path; + GetLocationPath(&path); + return service()->file()->GetSourceLocation(path, out_location); +} + +bool ServiceDescriptor::GetSourceLocation(SourceLocation* out_location) const { + vector<int> path; + GetLocationPath(&path); + return file()->GetSourceLocation(path, out_location); +} + +bool EnumValueDescriptor::GetSourceLocation( + SourceLocation* out_location) const { + vector<int> path; + GetLocationPath(&path); + return type()->file()->GetSourceLocation(path, out_location); +} + +void Descriptor::GetLocationPath(vector<int>* output) const { + if (containing_type()) { + containing_type()->GetLocationPath(output); + output->push_back(DescriptorProto::kNestedTypeFieldNumber); + output->push_back(index()); + } else { + output->push_back(FileDescriptorProto::kMessageTypeFieldNumber); + output->push_back(index()); + } +} + +void FieldDescriptor::GetLocationPath(vector<int>* output) const { + containing_type()->GetLocationPath(output); + output->push_back(DescriptorProto::kFieldFieldNumber); + output->push_back(index()); +} + +void EnumDescriptor::GetLocationPath(vector<int>* output) const { + if (containing_type()) { + containing_type()->GetLocationPath(output); + output->push_back(DescriptorProto::kEnumTypeFieldNumber); + output->push_back(index()); + } else { + output->push_back(FileDescriptorProto::kEnumTypeFieldNumber); + output->push_back(index()); + } +} + +void EnumValueDescriptor::GetLocationPath(vector<int>* output) const { + type()->GetLocationPath(output); + output->push_back(EnumDescriptorProto::kValueFieldNumber); + output->push_back(index()); +} + +void ServiceDescriptor::GetLocationPath(vector<int>* output) const { + output->push_back(FileDescriptorProto::kServiceFieldNumber); + output->push_back(index()); +} + +void MethodDescriptor::GetLocationPath(vector<int>* output) const { + service()->GetLocationPath(output); + output->push_back(ServiceDescriptorProto::kMethodFieldNumber); + output->push_back(index()); +} + // =================================================================== namespace { @@ -1859,6 +2154,7 @@ class DescriptorBuilder { string filename_; FileDescriptor* file_; FileDescriptorTables* file_tables_; + set<const FileDescriptor*> dependencies_; // If LookupSymbol() finds a symbol that is in a file which is not a declared // dependency of this file, it will fail, but will set @@ -1888,6 +2184,10 @@ class DescriptorBuilder { // nested package within package_name. bool IsInPackage(const FileDescriptor* file, const string& package_name); + // Helper function which finds all public dependencies of the given file, and + // stores the them in the dependencies_ set in the builder. + void RecordPublicDependencies(const FileDescriptor* file); + // Like tables_->FindSymbol(), but additionally: // - Search the pool's underlay if not found in tables_. // - Insure that the resulting Symbol is from one of the file's declared @@ -1898,6 +2198,10 @@ class DescriptorBuilder { // file's declared dependencies. Symbol FindSymbolNotEnforcingDeps(const string& name); + // This implements the body of FindSymbolNotEnforcingDeps(). + Symbol FindSymbolNotEnforcingDepsHelper(const DescriptorPool* pool, + const string& name); + // Like FindSymbol(), but looks up the name relative to some other symbol // name. This first searches siblings of relative_to, then siblings of its // parents, etc. For example, LookupSymbol("foo.bar", "baz.qux.corge") makes @@ -2189,6 +2493,7 @@ class DescriptorBuilder { void ValidateMapKey(FieldDescriptor* field, const FieldDescriptorProto& proto); + }; const FileDescriptor* DescriptorPool::BuildFile( @@ -2275,31 +2580,48 @@ bool DescriptorBuilder::IsInPackage(const FileDescriptor* file, file->package()[package_name.size()] == '.'); } -Symbol DescriptorBuilder::FindSymbolNotEnforcingDeps(const string& name) { - Symbol result; +void DescriptorBuilder::RecordPublicDependencies(const FileDescriptor* file) { + if (file == NULL || !dependencies_.insert(file).second) return; + for (int i = 0; file != NULL && i < file->public_dependency_count(); i++) { + RecordPublicDependencies(file->public_dependency(i)); + } +} - // We need to search our pool and all its underlays. - const DescriptorPool* pool = pool_; - while (true) { - // If we are looking at an underlay, we must lock its mutex_, since we are - // accessing the underlay's tables_ dircetly. - MutexLockMaybe lock((pool == pool_) ? NULL : pool->mutex_); +Symbol DescriptorBuilder::FindSymbolNotEnforcingDepsHelper( + const DescriptorPool* pool, const string& name) { + // If we are looking at an underlay, we must lock its mutex_, since we are + // accessing the underlay's tables_ directly. + MutexLockMaybe lock((pool == pool_) ? NULL : pool->mutex_); + + Symbol result = pool->tables_->FindSymbol(name); + if (result.IsNull() && pool->underlay_ != NULL) { + // Symbol not found; check the underlay. + result = FindSymbolNotEnforcingDepsHelper(pool->underlay_, name); + } - // Note that we don't have to check fallback_database_ here because the - // symbol has to be in one of its file's direct dependencies, and we have - // already loaded those by the time we get here. - result = pool->tables_->FindSymbol(name); - if (!result.IsNull()) break; - if (pool->underlay_ == NULL) return kNullSymbol; - pool = pool->underlay_; + if (result.IsNull()) { + // In theory, we shouldn't need to check fallback_database_ because the + // symbol should be in one of its file's direct dependencies, and we have + // already loaded those by the time we get here. But we check anyway so + // that we can generate better error message when dependencies are missing + // (i.e., "missing dependency" rather than "type is not defined"). + if (pool->TryFindSymbolInFallbackDatabase(name)) { + result = pool->tables_->FindSymbol(name); + } } return result; } +Symbol DescriptorBuilder::FindSymbolNotEnforcingDeps(const string& name) { + return FindSymbolNotEnforcingDepsHelper(pool_, name); +} + Symbol DescriptorBuilder::FindSymbol(const string& name) { Symbol result = FindSymbolNotEnforcingDeps(name); + if (result.IsNull()) return result; + if (!pool_->enforce_dependencies_) { // Hack for CompilerUpgrader. return result; @@ -2308,10 +2630,7 @@ Symbol DescriptorBuilder::FindSymbol(const string& name) { // Only find symbols which were defined in this file or one of its // dependencies. const FileDescriptor* file = result.GetFile(); - if (file == file_) return result; - for (int i = 0; i < file_->dependency_count(); i++) { - if (file == file_->dependency(i)) return result; - } + if (file == file_ || dependencies_.count(file) > 0) return result; if (result.type == Symbol::PACKAGE) { // Arg, this is overcomplicated. The symbol is a package name. It could @@ -2322,12 +2641,10 @@ Symbol DescriptorBuilder::FindSymbol(const string& name) { // dependency also defines the same package. We can't really rule out this // symbol unless none of the dependencies define it. if (IsInPackage(file_, name)) return result; - for (int i = 0; i < file_->dependency_count(); i++) { + for (set<const FileDescriptor*>::const_iterator it = dependencies_.begin(); + it != dependencies_.end(); ++it) { // Note: A dependency may be NULL if it was not found or had errors. - if (file_->dependency(i) != NULL && - IsInPackage(file_->dependency(i), name)) { - return result; - } + if (*it != NULL && IsInPackage(*it, name)) return result; } } @@ -2448,6 +2765,8 @@ Symbol DescriptorBuilder::NewPlaceholder(const string& name, FileDescriptor* placeholder_file = tables_->Allocate<FileDescriptor>(); memset(placeholder_file, 0, sizeof(*placeholder_file)); + placeholder_file->source_code_info_ = &SourceCodeInfo::default_instance(); + placeholder_file->name_ = tables_->AllocateString(*placeholder_full_name + ".placeholder.proto"); placeholder_file->package_ = placeholder_package; @@ -2757,11 +3076,19 @@ const FileDescriptor* DescriptorBuilder::BuildFile( } // Checkpoint the tables so that we can roll back if something goes wrong. - tables_->Checkpoint(); + tables_->AddCheckpoint(); FileDescriptor* result = tables_->Allocate<FileDescriptor>(); file_ = result; + if (proto.has_source_code_info()) { + SourceCodeInfo *info = tables_->AllocateMessage<SourceCodeInfo>(); + info->CopyFrom(proto.source_code_info()); + result->source_code_info_ = info; + } else { + result->source_code_info_ = &SourceCodeInfo::default_instance(); + } + file_tables_ = tables_->AllocateFileTables(); file_->tables_ = file_tables_; @@ -2788,7 +3115,7 @@ const FileDescriptor* DescriptorBuilder::BuildFile( "A file with this name is already in the pool."); // Bail out early so that if this is actually the exact same file, we // don't end up reporting that every single symbol is already defined. - tables_->Rollback(); + tables_->RollbackToLastCheckpoint(); return NULL; } if (!result->package().empty()) { @@ -2833,6 +3160,45 @@ const FileDescriptor* DescriptorBuilder::BuildFile( result->dependencies_[i] = dependency; } + // Check public dependencies. + int public_dependency_count = 0; + result->public_dependencies_ = tables_->AllocateArray<int>( + proto.public_dependency_size()); + for (int i = 0; i < proto.public_dependency_size(); i++) { + // Only put valid public dependency indexes. + int index = proto.public_dependency(i); + if (index >= 0 && index < proto.dependency_size()) { + result->public_dependencies_[public_dependency_count++] = index; + } else { + AddError(proto.name(), proto, + DescriptorPool::ErrorCollector::OTHER, + "Invalid public dependency index."); + } + } + result->public_dependency_count_ = public_dependency_count; + + // Build dependency set + dependencies_.clear(); + for (int i = 0; i < result->dependency_count(); i++) { + RecordPublicDependencies(result->dependency(i)); + } + + // Check weak dependencies. + int weak_dependency_count = 0; + result->weak_dependencies_ = tables_->AllocateArray<int>( + proto.weak_dependency_size()); + for (int i = 0; i < proto.weak_dependency_size(); i++) { + int index = proto.weak_dependency(i); + if (index >= 0 && index < proto.dependency_size()) { + result->weak_dependencies_[weak_dependency_count++] = index; + } else { + AddError(proto.name(), proto, + DescriptorPool::ErrorCollector::OTHER, + "Invalid weak dependency index."); + } + } + result->weak_dependency_count_ = weak_dependency_count; + // Convert children. BUILD_ARRAY(proto, result, message_type, BuildMessage , NULL); BUILD_ARRAY(proto, result, enum_type , BuildEnum , NULL); @@ -2870,10 +3236,10 @@ const FileDescriptor* DescriptorBuilder::BuildFile( } if (had_errors_) { - tables_->Rollback(); + tables_->RollbackToLastCheckpoint(); return NULL; } else { - tables_->Checkpoint(); + tables_->ClearLastCheckpoint(); return result; } } @@ -3065,7 +3431,7 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, UnescapeCEscapeString(proto.default_value())); } else { result->default_value_string_ = - tables_->AllocateString(proto.default_value()); + tables_->AllocateString(proto.default_value()); } break; case FieldDescriptor::CPPTYPE_MESSAGE: @@ -3126,7 +3492,15 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, if (result->number() <= 0) { AddError(result->full_name(), proto, DescriptorPool::ErrorCollector::NUMBER, "Field numbers must be positive integers."); - } else if (result->number() > FieldDescriptor::kMaxNumber) { + } else if (!is_extension && result->number() > FieldDescriptor::kMaxNumber) { + // Only validate that the number is within the valid field range if it is + // not an extension. Since extension numbers are validated with the + // extendee's valid set of extension numbers, and those are in turn + // validated against the max allowed number, the check is unnecessary for + // extension fields. + // This avoids cross-linking issues that arise when attempting to check if + // the extendee is a message_set_wire_format message, which has a higher max + // on extension numbers. AddError(result->full_name(), proto, DescriptorPool::ErrorCollector::NUMBER, strings::Substitute("Field numbers cannot be greater than $0.", FieldDescriptor::kMaxNumber)); @@ -3181,12 +3555,10 @@ void DescriptorBuilder::BuildExtensionRange( "Extension numbers must be positive integers."); } - if (result->end > FieldDescriptor::kMaxNumber + 1) { - AddError(parent->full_name(), proto, - DescriptorPool::ErrorCollector::NUMBER, - strings::Substitute("Extension numbers cannot be greater than $0.", - FieldDescriptor::kMaxNumber)); - } + // Checking of the upper bound of the extension range is deferred until after + // options interpreting. This allows messages with message_set_wire_format to + // have extensions beyond FieldDescriptor::kMaxNumber, since the extension + // numbers are actually used as int32s in the message_set_wire_format. if (result->start >= result->end) { AddError(parent->full_name(), proto, @@ -3681,6 +4053,20 @@ void DescriptorBuilder::ValidateMessageOptions(Descriptor* message, VALIDATE_OPTIONS_FROM_ARRAY(message, nested_type, Message); VALIDATE_OPTIONS_FROM_ARRAY(message, enum_type, Enum); VALIDATE_OPTIONS_FROM_ARRAY(message, extension, Field); + + const int64 max_extension_range = + static_cast<int64>(message->options().message_set_wire_format() ? + kint32max : + FieldDescriptor::kMaxNumber); + for (int i = 0; i < message->extension_range_count(); ++i) { + if (message->extension_range(i)->end > max_extension_range + 1) { + AddError( + message->full_name(), proto.extension_range(i), + DescriptorPool::ErrorCollector::NUMBER, + strings::Substitute("Extension numbers cannot be greater than $0.", + max_extension_range)); + } + } } void DescriptorBuilder::ValidateFieldOptions(FieldDescriptor* field, @@ -3689,6 +4075,15 @@ void DescriptorBuilder::ValidateFieldOptions(FieldDescriptor* field, ValidateMapKey(field, proto); } + // Only message type fields may be lazy. + if (field->options().lazy()) { + if (field->type() != FieldDescriptor::TYPE_MESSAGE) { + AddError(field->full_name(), proto, + DescriptorPool::ErrorCollector::TYPE, + "[lazy = true] can only be specified for submessage fields."); + } + } + // Only repeated primitive fields may be packed. if (field->options().packed() && !field->is_packable()) { AddError( @@ -3727,11 +4122,27 @@ void DescriptorBuilder::ValidateFieldOptions(FieldDescriptor* field, "files. Note that you cannot extend a non-lite type to contain " "a lite type, but the reverse is allowed."); } + } void DescriptorBuilder::ValidateEnumOptions(EnumDescriptor* enm, const EnumDescriptorProto& proto) { VALIDATE_OPTIONS_FROM_ARRAY(enm, value, EnumValue); + if (!enm->options().allow_alias()) { + map<int, string> used_values; + for (int i = 0; i < enm->value_count(); ++i) { + const EnumValueDescriptor* enum_value = enm->value(i); + if (used_values.find(enum_value->number()) != used_values.end()) { + AddError(enm->full_name(), proto, + DescriptorPool::ErrorCollector::NUMBER, + "\"" + enum_value->full_name() + + "\" uses the same enum value as \"" + + used_values[enum_value->number()] + "\""); + } else { + used_values[enum_value->number()] = enum_value->full_name(); + } + } + } } void DescriptorBuilder::ValidateEnumValueOptions( @@ -3811,6 +4222,7 @@ void DescriptorBuilder::ValidateMapKey(FieldDescriptor* field, field->experimental_map_key_ = key_field; } + #undef VALIDATE_OPTIONS_FROM_ARRAY // ------------------------------------------------------------------- @@ -3901,9 +4313,11 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( // file we're currently building. The descriptor should be there as long as // the file we're building imported "google/protobuf/descriptors.proto". - // Note that we use DescriptorBuilder::FindSymbol(), not + // Note that we use DescriptorBuilder::FindSymbolNotEnforcingDeps(), not // DescriptorPool::FindMessageTypeByName() because we're already holding the - // pool's mutex, and the latter method locks it again. + // pool's mutex, and the latter method locks it again. We don't use + // FindSymbol() because files that use custom options only need to depend on + // the file that defines the option, not descriptor.proto itself. Symbol symbol = builder_->FindSymbolNotEnforcingDeps( options->GetDescriptor()->full_name()); if (!symbol.IsNull() && symbol.type == Symbol::MESSAGE) { @@ -3939,8 +4353,8 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( // DescriptorPool::FindExtensionByName(), for two reasons: 1) It allows // relative lookups, and 2) because we're already holding the pool's // mutex, and the latter method locks it again. - Symbol symbol = builder_->LookupSymbol(name_part, - options_to_interpret_->name_scope); + symbol = builder_->LookupSymbol(name_part, + options_to_interpret_->name_scope); if (!symbol.IsNull() && symbol.type == Symbol::FIELD) { field = symbol.field_descriptor; } @@ -4344,14 +4758,32 @@ class DescriptorBuilder::OptionInterpreter::AggregateOptionFinder virtual const FieldDescriptor* FindExtension( Message* message, const string& name) const { assert_mutex_held(builder_->pool_); + const Descriptor* descriptor = message->GetDescriptor(); Symbol result = builder_->LookupSymbolNoPlaceholder( - name, message->GetDescriptor()->full_name()); + name, descriptor->full_name()); if (result.type == Symbol::FIELD && result.field_descriptor->is_extension()) { return result.field_descriptor; - } else { - return NULL; + } else if (result.type == Symbol::MESSAGE && + descriptor->options().message_set_wire_format()) { + const Descriptor* foreign_type = result.descriptor; + // The text format allows MessageSet items to be specified using + // the type name, rather than the extension identifier. If the symbol + // lookup returned a Message, and the enclosing Message has + // message_set_wire_format = true, then return the message set + // extension, if one exists. + for (int i = 0; i < foreign_type->extension_count(); i++) { + const FieldDescriptor* extension = foreign_type->extension(i); + if (extension->containing_type() == descriptor && + extension->type() == FieldDescriptor::TYPE_MESSAGE && + extension->is_optional() && + extension->message_type() == foreign_type) { + // Found it. + return extension; + } + } } + return NULL; } }; @@ -4409,7 +4841,13 @@ bool DescriptorBuilder::OptionInterpreter::SetAggregateOption( } else { string serial; dynamic->SerializeToString(&serial); // Never fails - unknown_fields->AddLengthDelimited(option_field->number(), serial); + if (option_field->type() == FieldDescriptor::TYPE_MESSAGE) { + unknown_fields->AddLengthDelimited(option_field->number(), serial); + } else { + GOOGLE_CHECK_EQ(option_field->type(), FieldDescriptor::TYPE_GROUP); + UnknownFieldSet* group = unknown_fields->AddGroup(option_field->number()); + group->ParseFromString(serial); + } return true; } } |