diff options
Diffstat (limited to 'python/google/protobuf/descriptor_pool.py')
| -rw-r--r-- | python/google/protobuf/descriptor_pool.py | 484 |
1 files changed, 396 insertions, 88 deletions
diff --git a/python/google/protobuf/descriptor_pool.py b/python/google/protobuf/descriptor_pool.py index 3e80795c..8983f76f 100644 --- a/python/google/protobuf/descriptor_pool.py +++ b/python/google/protobuf/descriptor_pool.py @@ -57,12 +57,15 @@ directly instead of this class. __author__ = 'matthewtoia@google.com (Matt Toia)' +import collections +import warnings + from google.protobuf import descriptor from google.protobuf import descriptor_database from google.protobuf import text_encoding -_USE_C_DESCRIPTORS = descriptor._USE_C_DESCRIPTORS +_USE_C_DESCRIPTORS = descriptor._USE_C_DESCRIPTORS # pylint: disable=protected-access def _NormalizeFullyQualifiedName(name): @@ -80,6 +83,22 @@ def _NormalizeFullyQualifiedName(name): return name.lstrip('.') +def _OptionsOrNone(descriptor_proto): + """Returns the value of the field `options`, or None if it is not set.""" + if descriptor_proto.HasField('options'): + return descriptor_proto.options + else: + return None + + +def _IsMessageSetExtension(field): + return (field.is_extension and + field.containing_type.has_options and + field.containing_type.GetOptions().message_set_wire_format and + field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and + field.label == descriptor.FieldDescriptor.LABEL_OPTIONAL) + + class DescriptorPool(object): """A collection of protobufs dynamically constructed by descriptor protos.""" @@ -106,7 +125,40 @@ class DescriptorPool(object): self._descriptor_db = descriptor_db self._descriptors = {} self._enum_descriptors = {} + self._service_descriptors = {} self._file_descriptors = {} + self._toplevel_extensions = {} + # TODO(jieluo): Remove _file_desc_by_toplevel_extension after + # maybe year 2020 for compatibility issue (with 3.4.1 only). + self._file_desc_by_toplevel_extension = {} + # We store extensions in two two-level mappings: The first key is the + # descriptor of the message being extended, the second key is the extension + # full name or its tag number. + self._extensions_by_name = collections.defaultdict(dict) + self._extensions_by_number = collections.defaultdict(dict) + + def _CheckConflictRegister(self, desc): + """Check if the descriptor name conflicts with another of the same name. + + Args: + desc: Descriptor of a message, enum, service or extension. + """ + desc_name = desc.full_name + for register, descriptor_type in [ + (self._descriptors, descriptor.Descriptor), + (self._enum_descriptors, descriptor.EnumDescriptor), + (self._service_descriptors, descriptor.ServiceDescriptor), + (self._toplevel_extensions, descriptor.FieldDescriptor)]: + if desc_name in register: + file_name = register[desc_name].file.name + if not isinstance(desc, descriptor_type) or ( + file_name != desc.file.name): + warn_msg = ('Conflict register for file "' + desc.file.name + + '": ' + desc_name + + ' is already defined in file "' + + file_name + '"') + warnings.warn(warn_msg, RuntimeWarning) + return def Add(self, file_desc_proto): """Adds the FileDescriptorProto and its types to this pool. @@ -144,13 +196,15 @@ class DescriptorPool(object): if not isinstance(desc, descriptor.Descriptor): raise TypeError('Expected instance of descriptor.Descriptor.') + self._CheckConflictRegister(desc) + self._descriptors[desc.full_name] = desc - self.AddFileDescriptor(desc.file) + self._AddFileDescriptor(desc.file) def AddEnumDescriptor(self, enum_desc): """Adds an EnumDescriptor to the pool. - This method also registers the FileDescriptor associated with the message. + This method also registers the FileDescriptor associated with the enum. Args: enum_desc: An EnumDescriptor. @@ -159,8 +213,65 @@ class DescriptorPool(object): if not isinstance(enum_desc, descriptor.EnumDescriptor): raise TypeError('Expected instance of descriptor.EnumDescriptor.') + self._CheckConflictRegister(enum_desc) self._enum_descriptors[enum_desc.full_name] = enum_desc - self.AddFileDescriptor(enum_desc.file) + self._AddFileDescriptor(enum_desc.file) + + def AddServiceDescriptor(self, service_desc): + """Adds a ServiceDescriptor to the pool. + + Args: + service_desc: A ServiceDescriptor. + """ + + if not isinstance(service_desc, descriptor.ServiceDescriptor): + raise TypeError('Expected instance of descriptor.ServiceDescriptor.') + + self._CheckConflictRegister(service_desc) + self._service_descriptors[service_desc.full_name] = service_desc + + def AddExtensionDescriptor(self, extension): + """Adds a FieldDescriptor describing an extension to the pool. + + Args: + extension: A FieldDescriptor. + + Raises: + AssertionError: when another extension with the same number extends the + same message. + TypeError: when the specified extension is not a + descriptor.FieldDescriptor. + """ + if not (isinstance(extension, descriptor.FieldDescriptor) and + extension.is_extension): + raise TypeError('Expected an extension descriptor.') + + if extension.extension_scope is None: + self._CheckConflictRegister(extension) + self._toplevel_extensions[extension.full_name] = extension + + try: + existing_desc = self._extensions_by_number[ + extension.containing_type][extension.number] + except KeyError: + pass + else: + if extension is not existing_desc: + raise AssertionError( + 'Extensions "%s" and "%s" both try to extend message type "%s" ' + 'with field number %d.' % + (extension.full_name, existing_desc.full_name, + extension.containing_type.full_name, extension.number)) + + self._extensions_by_number[extension.containing_type][ + extension.number] = extension + self._extensions_by_name[extension.containing_type][ + extension.full_name] = extension + + # Also register MessageSet extensions with the type name. + if _IsMessageSetExtension(extension): + self._extensions_by_name[extension.containing_type][ + extension.message_type.full_name] = extension def AddFileDescriptor(self, file_desc): """Adds a FileDescriptor to the pool, non-recursively. @@ -172,6 +283,24 @@ class DescriptorPool(object): file_desc: A FileDescriptor. """ + self._AddFileDescriptor(file_desc) + # TODO(jieluo): This is a temporary solution for FieldDescriptor.file. + # FieldDescriptor.file is added in code gen. Remove this solution after + # maybe 2020 for compatibility reason (with 3.4.1 only). + for extension in file_desc.extensions_by_name.values(): + self._file_desc_by_toplevel_extension[ + extension.full_name] = file_desc + + def _AddFileDescriptor(self, file_desc): + """Adds a FileDescriptor to the pool, non-recursively. + + If the FileDescriptor contains messages or enums, the caller must explicitly + register them. + + Args: + file_desc: A FileDescriptor. + """ + if not isinstance(file_desc, descriptor.FileDescriptor): raise TypeError('Expected instance of descriptor.FileDescriptor.') self._file_descriptors[file_desc.name] = file_desc @@ -186,7 +315,7 @@ class DescriptorPool(object): A FileDescriptor for the named file. Raises: - KeyError: if the file can not be found in the pool. + KeyError: if the file cannot be found in the pool. """ try: @@ -215,7 +344,7 @@ class DescriptorPool(object): A FileDescriptor that contains the specified symbol. Raises: - KeyError: if the file can not be found in the pool. + KeyError: if the file cannot be found in the pool. """ symbol = _NormalizeFullyQualifiedName(symbol) @@ -230,15 +359,28 @@ class DescriptorPool(object): pass try: - file_proto = self._internal_db.FindFileContainingSymbol(symbol) - except KeyError as error: - if self._descriptor_db: - file_proto = self._descriptor_db.FindFileContainingSymbol(symbol) - else: - raise error - if not file_proto: + return self._service_descriptors[symbol].file + except KeyError: + pass + + try: + return self._FindFileContainingSymbolInDb(symbol) + except KeyError: + pass + + try: + return self._file_desc_by_toplevel_extension[symbol] + except KeyError: + pass + + # Try nested extensions inside a message. + message_name, _, extension_name = symbol.rpartition('.') + try: + message = self.FindMessageTypeByName(message_name) + assert message.extensions_by_name[extension_name] + return message.file + except KeyError: raise KeyError('Cannot find a file containing %s' % symbol) - return self._ConvertFileProtoToFileDescriptor(file_proto) def FindMessageTypeByName(self, full_name): """Loads the named descriptor from the pool. @@ -248,11 +390,14 @@ class DescriptorPool(object): Returns: The descriptor for the named type. + + Raises: + KeyError: if the message cannot be found in the pool. """ full_name = _NormalizeFullyQualifiedName(full_name) if full_name not in self._descriptors: - self.FindFileContainingSymbol(full_name) + self._FindFileContainingSymbolInDb(full_name) return self._descriptors[full_name] def FindEnumTypeByName(self, full_name): @@ -263,11 +408,14 @@ class DescriptorPool(object): Returns: The enum descriptor for the named type. + + Raises: + KeyError: if the enum cannot be found in the pool. """ full_name = _NormalizeFullyQualifiedName(full_name) if full_name not in self._enum_descriptors: - self.FindFileContainingSymbol(full_name) + self._FindFileContainingSymbolInDb(full_name) return self._enum_descriptors[full_name] def FindFieldByName(self, full_name): @@ -278,12 +426,32 @@ class DescriptorPool(object): Returns: The field descriptor for the named field. + + Raises: + KeyError: if the field cannot be found in the pool. """ full_name = _NormalizeFullyQualifiedName(full_name) message_name, _, field_name = full_name.rpartition('.') message_descriptor = self.FindMessageTypeByName(message_name) return message_descriptor.fields_by_name[field_name] + def FindOneofByName(self, full_name): + """Loads the named oneof descriptor from the pool. + + Args: + full_name: The full name of the oneof descriptor to load. + + Returns: + The oneof descriptor for the named oneof. + + Raises: + KeyError: if the oneof cannot be found in the pool. + """ + full_name = _NormalizeFullyQualifiedName(full_name) + message_name, _, oneof_name = full_name.rpartition('.') + message_descriptor = self.FindMessageTypeByName(message_name) + return message_descriptor.oneofs_by_name[oneof_name] + def FindExtensionByName(self, full_name): """Loads the named extension descriptor from the pool. @@ -292,17 +460,101 @@ class DescriptorPool(object): Returns: A FieldDescriptor, describing the named extension. + + Raises: + KeyError: if the extension cannot be found in the pool. """ full_name = _NormalizeFullyQualifiedName(full_name) + try: + # The proto compiler does not give any link between the FileDescriptor + # and top-level extensions unless the FileDescriptorProto is added to + # the DescriptorDatabase, but this can impact memory usage. + # So we registered these extensions by name explicitly. + return self._toplevel_extensions[full_name] + except KeyError: + pass message_name, _, extension_name = full_name.rpartition('.') try: # Most extensions are nested inside a message. scope = self.FindMessageTypeByName(message_name) except KeyError: # Some extensions are defined at file scope. - scope = self.FindFileContainingSymbol(full_name) + scope = self._FindFileContainingSymbolInDb(full_name) return scope.extensions_by_name[extension_name] + def FindExtensionByNumber(self, message_descriptor, number): + """Gets the extension of the specified message with the specified number. + + Extensions have to be registered to this pool by calling + AddExtensionDescriptor. + + Args: + message_descriptor: descriptor of the extended message. + number: integer, number of the extension field. + + Returns: + A FieldDescriptor describing the extension. + + Raises: + KeyError: when no extension with the given number is known for the + specified message. + """ + return self._extensions_by_number[message_descriptor][number] + + def FindAllExtensions(self, message_descriptor): + """Gets all the known extension of a given message. + + Extensions have to be registered to this pool by calling + AddExtensionDescriptor. + + Args: + message_descriptor: descriptor of the extended message. + + Returns: + A list of FieldDescriptor describing the extensions. + """ + return list(self._extensions_by_number[message_descriptor].values()) + + def FindServiceByName(self, full_name): + """Loads the named service descriptor from the pool. + + Args: + full_name: The full name of the service descriptor to load. + + Returns: + The service descriptor for the named service. + + Raises: + KeyError: if the service cannot be found in the pool. + """ + full_name = _NormalizeFullyQualifiedName(full_name) + if full_name not in self._service_descriptors: + self._FindFileContainingSymbolInDb(full_name) + return self._service_descriptors[full_name] + + def _FindFileContainingSymbolInDb(self, symbol): + """Finds the file in descriptor DB containing the specified symbol. + + Args: + symbol: The name of the symbol to search for. + + Returns: + A FileDescriptor that contains the specified symbol. + + Raises: + KeyError: if the file cannot be found in the descriptor database. + """ + try: + file_proto = self._internal_db.FindFileContainingSymbol(symbol) + except KeyError as error: + if self._descriptor_db: + file_proto = self._descriptor_db.FindFileContainingSymbol(symbol) + else: + raise error + if not file_proto: + raise KeyError('Cannot find a file containing %s' % symbol) + return self._ConvertFileProtoToFileDescriptor(file_proto) + def _ConvertFileProtoToFileDescriptor(self, file_proto): """Creates a FileDescriptor from a proto or returns a cached copy. @@ -319,78 +571,69 @@ class DescriptorPool(object): if file_proto.name not in self._file_descriptors: built_deps = list(self._GetDeps(file_proto.dependency)) direct_deps = [self.FindFileByName(n) for n in file_proto.dependency] + public_deps = [direct_deps[i] for i in file_proto.public_dependency] file_descriptor = descriptor.FileDescriptor( pool=self, name=file_proto.name, package=file_proto.package, syntax=file_proto.syntax, - options=file_proto.options, + options=_OptionsOrNone(file_proto), serialized_pb=file_proto.SerializeToString(), - dependencies=direct_deps) - if _USE_C_DESCRIPTORS: - # When using C++ descriptors, all objects defined in the file were added - # to the C++ database when the FileDescriptor was built above. - # Just add them to this descriptor pool. - def _AddMessageDescriptor(message_desc): - self._descriptors[message_desc.full_name] = message_desc - for nested in message_desc.nested_types: - _AddMessageDescriptor(nested) - for enum_type in message_desc.enum_types: - _AddEnumDescriptor(enum_type) - def _AddEnumDescriptor(enum_desc): - self._enum_descriptors[enum_desc.full_name] = enum_desc - for message_type in file_descriptor.message_types_by_name.values(): - _AddMessageDescriptor(message_type) - for enum_type in file_descriptor.enum_types_by_name.values(): - _AddEnumDescriptor(enum_type) + dependencies=direct_deps, + public_dependencies=public_deps) + scope = {} + + # This loop extracts all the message and enum types from all the + # dependencies of the file_proto. This is necessary to create the + # scope of available message types when defining the passed in + # file proto. + for dependency in built_deps: + scope.update(self._ExtractSymbols( + dependency.message_types_by_name.values())) + scope.update((_PrefixWithDot(enum.full_name), enum) + for enum in dependency.enum_types_by_name.values()) + + for message_type in file_proto.message_type: + message_desc = self._ConvertMessageDescriptor( + message_type, file_proto.package, file_descriptor, scope, + file_proto.syntax) + file_descriptor.message_types_by_name[message_desc.name] = ( + message_desc) + + for enum_type in file_proto.enum_type: + file_descriptor.enum_types_by_name[enum_type.name] = ( + self._ConvertEnumDescriptor(enum_type, file_proto.package, + file_descriptor, None, scope)) + + for index, extension_proto in enumerate(file_proto.extension): + extension_desc = self._MakeFieldDescriptor( + extension_proto, file_proto.package, index, file_descriptor, + is_extension=True) + extension_desc.containing_type = self._GetTypeFromScope( + file_descriptor.package, extension_proto.extendee, scope) + self._SetFieldType(extension_proto, extension_desc, + file_descriptor.package, scope) + file_descriptor.extensions_by_name[extension_desc.name] = ( + extension_desc) + + for desc_proto in file_proto.message_type: + self._SetAllFieldTypes(file_proto.package, desc_proto, scope) + + if file_proto.package: + desc_proto_prefix = _PrefixWithDot(file_proto.package) else: - scope = {} - - # This loop extracts all the message and enum types from all the - # dependencies of the file_proto. This is necessary to create the - # scope of available message types when defining the passed in - # file proto. - for dependency in built_deps: - scope.update(self._ExtractSymbols( - dependency.message_types_by_name.values())) - scope.update((_PrefixWithDot(enum.full_name), enum) - for enum in dependency.enum_types_by_name.values()) - - for message_type in file_proto.message_type: - message_desc = self._ConvertMessageDescriptor( - message_type, file_proto.package, file_descriptor, scope, - file_proto.syntax) - file_descriptor.message_types_by_name[message_desc.name] = ( - message_desc) - - for enum_type in file_proto.enum_type: - file_descriptor.enum_types_by_name[enum_type.name] = ( - self._ConvertEnumDescriptor(enum_type, file_proto.package, - file_descriptor, None, scope)) - - for index, extension_proto in enumerate(file_proto.extension): - extension_desc = self._MakeFieldDescriptor( - extension_proto, file_proto.package, index, is_extension=True) - extension_desc.containing_type = self._GetTypeFromScope( - file_descriptor.package, extension_proto.extendee, scope) - self._SetFieldType(extension_proto, extension_desc, - file_descriptor.package, scope) - file_descriptor.extensions_by_name[extension_desc.name] = ( - extension_desc) - - for desc_proto in file_proto.message_type: - self._SetAllFieldTypes(file_proto.package, desc_proto, scope) - - if file_proto.package: - desc_proto_prefix = _PrefixWithDot(file_proto.package) - else: - desc_proto_prefix = '' + desc_proto_prefix = '' + + for desc_proto in file_proto.message_type: + desc = self._GetTypeFromScope( + desc_proto_prefix, desc_proto.name, scope) + file_descriptor.message_types_by_name[desc_proto.name] = desc - for desc_proto in file_proto.message_type: - desc = self._GetTypeFromScope( - desc_proto_prefix, desc_proto.name, scope) - file_descriptor.message_types_by_name[desc_proto.name] = desc + for index, service_proto in enumerate(file_proto.service): + file_descriptor.services_by_name[service_proto.name] = ( + self._MakeServiceDescriptor(service_proto, index, scope, + file_proto.package, file_descriptor)) self.Add(file_proto) self._file_descriptors[file_proto.name] = file_descriptor @@ -406,6 +649,7 @@ class DescriptorPool(object): package: The package the proto should be located in. file_desc: The file containing this message. scope: Dict mapping short and full symbols to message and enum types. + syntax: string indicating syntax of the file ("proto2" or "proto3") Returns: The added descriptor. @@ -431,15 +675,15 @@ class DescriptorPool(object): enums = [ self._ConvertEnumDescriptor(enum, desc_name, file_desc, None, scope) for enum in desc_proto.enum_type] - fields = [self._MakeFieldDescriptor(field, desc_name, index) + fields = [self._MakeFieldDescriptor(field, desc_name, index, file_desc) for index, field in enumerate(desc_proto.field)] extensions = [ - self._MakeFieldDescriptor(extension, desc_name, index, + self._MakeFieldDescriptor(extension, desc_name, index, file_desc, is_extension=True) for index, extension in enumerate(desc_proto.extension)] oneofs = [ descriptor.OneofDescriptor(desc.name, '.'.join((desc_name, desc.name)), - index, None, []) + index, None, [], desc.options) for index, desc in enumerate(desc_proto.oneof_decl)] extension_ranges = [(r.start, r.end) for r in desc_proto.extension_range] if extension_ranges: @@ -456,7 +700,7 @@ class DescriptorPool(object): nested_types=nested, enum_types=enums, extensions=extensions, - options=desc_proto.options, + options=_OptionsOrNone(desc_proto), is_extendable=is_extendable, extension_ranges=extension_ranges, file=file_desc, @@ -474,6 +718,7 @@ class DescriptorPool(object): fields[field_index].containing_oneof = oneofs[oneof_index] scope[_PrefixWithDot(desc_name)] = desc + self._CheckConflictRegister(desc) self._descriptors[desc_name] = desc return desc @@ -510,13 +755,14 @@ class DescriptorPool(object): file=file_desc, values=values, containing_type=containing_type, - options=enum_proto.options) + options=_OptionsOrNone(enum_proto)) scope['.%s' % enum_name] = desc + self._CheckConflictRegister(desc) self._enum_descriptors[enum_name] = desc return desc def _MakeFieldDescriptor(self, field_proto, message_name, index, - is_extension=False): + file_desc, is_extension=False): """Creates a field descriptor from a FieldDescriptorProto. For message and enum type fields, this method will do a look up @@ -529,6 +775,7 @@ class DescriptorPool(object): field_proto: The proto describing the field. message_name: The name of the containing message. index: Index of the field + file_desc: The file containing the field descriptor. is_extension: Indication that this field is for an extension. Returns: @@ -555,7 +802,8 @@ class DescriptorPool(object): default_value=None, is_extension=is_extension, extension_scope=None, - options=field_proto.options) + options=_OptionsOrNone(field_proto), + file=file_desc) def _SetAllFieldTypes(self, package, desc_proto, scope): """Sets all the descriptor's fields's types. @@ -674,9 +922,69 @@ class DescriptorPool(object): name=value_proto.name, index=index, number=value_proto.number, - options=value_proto.options, + options=_OptionsOrNone(value_proto), type=None) + def _MakeServiceDescriptor(self, service_proto, service_index, scope, + package, file_desc): + """Make a protobuf ServiceDescriptor given a ServiceDescriptorProto. + + Args: + service_proto: The descriptor_pb2.ServiceDescriptorProto protobuf message. + service_index: The index of the service in the File. + scope: Dict mapping short and full symbols to message and enum types. + package: Optional package name for the new message EnumDescriptor. + file_desc: The file containing the service descriptor. + + Returns: + The added descriptor. + """ + + if package: + service_name = '.'.join((package, service_proto.name)) + else: + service_name = service_proto.name + + methods = [self._MakeMethodDescriptor(method_proto, service_name, package, + scope, index) + for index, method_proto in enumerate(service_proto.method)] + desc = descriptor.ServiceDescriptor(name=service_proto.name, + full_name=service_name, + index=service_index, + methods=methods, + options=_OptionsOrNone(service_proto), + file=file_desc) + self._CheckConflictRegister(desc) + self._service_descriptors[service_name] = desc + return desc + + def _MakeMethodDescriptor(self, method_proto, service_name, package, scope, + index): + """Creates a method descriptor from a MethodDescriptorProto. + + Args: + method_proto: The proto describing the method. + service_name: The name of the containing service. + package: Optional package name to look up for types. + scope: Scope containing available types. + index: Index of the method in the service. + + Returns: + An initialized MethodDescriptor object. + """ + full_name = '.'.join((service_name, method_proto.name)) + input_type = self._GetTypeFromScope( + package, method_proto.input_type, scope) + output_type = self._GetTypeFromScope( + package, method_proto.output_type, scope) + return descriptor.MethodDescriptor(name=method_proto.name, + full_name=full_name, + index=index, + containing_service=None, + input_type=input_type, + output_type=output_type, + options=_OptionsOrNone(method_proto)) + def _ExtractSymbols(self, descriptors): """Pulls out all the symbols from descriptor protos. |