down-integrate internal changes

1 files changed, 145 insertions, 14 deletions

diff --git a/python/google/protobuf/internal/python_message.py b/python/google/protobuf/internal/python_message.py
index 54f584ae..ca9f7675 100755
--- a/python/google/protobuf/internal/python_message.py
+++ b/python/google/protobuf/internal/python_message.py
@@ -61,9 +61,11 @@ if sys.version_info[0] < 3:
   except ImportError:
     from StringIO import StringIO as BytesIO
   import copy_reg as copyreg
+  _basestring = basestring
 else:
   from io import BytesIO
   import copyreg
+  _basestring = str
 import struct
 import weakref
 
@@ -77,6 +79,7 @@ from google.protobuf.internal import type_checkers
 from google.protobuf.internal import wire_format
 from google.protobuf import descriptor as descriptor_mod
 from google.protobuf import message as message_mod
+from google.protobuf import symbol_database
 from google.protobuf import text_format
 
 _FieldDescriptor = descriptor_mod.FieldDescriptor
@@ -101,6 +104,7 @@ def InitMessage(descriptor, cls):
   for field in descriptor.fields:
     _AttachFieldHelpers(cls, field)
 
+  descriptor._concrete_class = cls  # pylint: disable=protected-access
   _AddEnumValues(descriptor, cls)
   _AddInitMethod(descriptor, cls)
   _AddPropertiesForFields(descriptor, cls)
@@ -198,12 +202,37 @@ def _IsMessageSetExtension(field):
           field.label == _FieldDescriptor.LABEL_OPTIONAL)
 
 
+def _IsMapField(field):
+  return (field.type == _FieldDescriptor.TYPE_MESSAGE and
+          field.message_type.has_options and
+          field.message_type.GetOptions().map_entry)
+
+
+def _IsMessageMapField(field):
+  value_type = field.message_type.fields_by_name["value"]
+  return value_type.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE
+
+
 def _AttachFieldHelpers(cls, field_descriptor):
   is_repeated = (field_descriptor.label == _FieldDescriptor.LABEL_REPEATED)
-  is_packed = (field_descriptor.has_options and
-               field_descriptor.GetOptions().packed)
-
-  if _IsMessageSetExtension(field_descriptor):
+  is_packable = (is_repeated and
+                 wire_format.IsTypePackable(field_descriptor.type))
+  if not is_packable:
+    is_packed = False
+  elif field_descriptor.containing_type.syntax == "proto2":
+    is_packed = (field_descriptor.has_options and
+                field_descriptor.GetOptions().packed)
+  else:
+    has_packed_false = (field_descriptor.has_options and
+                        field_descriptor.GetOptions().HasField("packed") and
+                        field_descriptor.GetOptions().packed == False)
+    is_packed = not has_packed_false
+  is_map_entry = _IsMapField(field_descriptor)
+
+  if is_map_entry:
+    field_encoder = encoder.MapEncoder(field_descriptor)
+    sizer = encoder.MapSizer(field_descriptor)
+  elif _IsMessageSetExtension(field_descriptor):
     field_encoder = encoder.MessageSetItemEncoder(field_descriptor.number)
     sizer = encoder.MessageSetItemSizer(field_descriptor.number)
   else:
@@ -228,9 +257,16 @@ def _AttachFieldHelpers(cls, field_descriptor):
     if field_descriptor.containing_oneof is not None:
       oneof_descriptor = field_descriptor
 
-    field_decoder = type_checkers.TYPE_TO_DECODER[decode_type](
-        field_descriptor.number, is_repeated, is_packed,
-        field_descriptor, field_descriptor._default_constructor)
+    if is_map_entry:
+      is_message_map = _IsMessageMapField(field_descriptor)
+
+      field_decoder = decoder.MapDecoder(
+          field_descriptor, _GetInitializeDefaultForMap(field_descriptor),
+          is_message_map)
+    else:
+      field_decoder = type_checkers.TYPE_TO_DECODER[decode_type](
+              field_descriptor.number, is_repeated, is_packed,
+              field_descriptor, field_descriptor._default_constructor)
 
     cls._decoders_by_tag[tag_bytes] = (field_decoder, oneof_descriptor)
 
@@ -265,6 +301,26 @@ def _AddEnumValues(descriptor, cls):
       setattr(cls, enum_value.name, enum_value.number)
 
 
+def _GetInitializeDefaultForMap(field):
+  if field.label != _FieldDescriptor.LABEL_REPEATED:
+    raise ValueError('map_entry set on non-repeated field %s' % (
+        field.name))
+  fields_by_name = field.message_type.fields_by_name
+  key_checker = type_checkers.GetTypeChecker(fields_by_name['key'])
+
+  value_field = fields_by_name['value']
+  if _IsMessageMapField(field):
+    def MakeMessageMapDefault(message):
+      return containers.MessageMap(
+          message._listener_for_children, value_field.message_type, key_checker)
+    return MakeMessageMapDefault
+  else:
+    value_checker = type_checkers.GetTypeChecker(value_field)
+    def MakePrimitiveMapDefault(message):
+      return containers.ScalarMap(
+          message._listener_for_children, key_checker, value_checker)
+    return MakePrimitiveMapDefault
+
 def _DefaultValueConstructorForField(field):
   """Returns a function which returns a default value for a field.
 
@@ -279,6 +335,9 @@ def _DefaultValueConstructorForField(field):
     value may refer back to |message| via a weak reference.
   """
 
+  if _IsMapField(field):
+    return _GetInitializeDefaultForMap(field)
+
   if field.label == _FieldDescriptor.LABEL_REPEATED:
     if field.has_default_value and field.default_value != []:
       raise ValueError('Repeated field default value not empty list: %s' % (
@@ -329,7 +388,22 @@ def _ReraiseTypeErrorWithFieldName(message_name, field_name):
 
 def _AddInitMethod(message_descriptor, cls):
   """Adds an __init__ method to cls."""
-  fields = message_descriptor.fields
+
+  def _GetIntegerEnumValue(enum_type, value):
+    """Convert a string or integer enum value to an integer.
+
+    If the value is a string, it is converted to the enum value in
+    enum_type with the same name.  If the value is not a string, it's
+    returned as-is.  (No conversion or bounds-checking is done.)
+    """
+    if isinstance(value, _basestring):
+      try:
+        return enum_type.values_by_name[value].number
+      except KeyError:
+        raise ValueError('Enum type %s: unknown label "%s"' % (
+            enum_type.full_name, value))
+    return value
+
   def init(self, **kwargs):
     self._cached_byte_size = 0
     self._cached_byte_size_dirty = len(kwargs) > 0
@@ -352,19 +426,37 @@ def _AddInitMethod(message_descriptor, cls):
       if field.label == _FieldDescriptor.LABEL_REPEATED:
         copy = field._default_constructor(self)
         if field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE:  # Composite
-          for val in field_value:
-            copy.add().MergeFrom(val)
+          if _IsMapField(field):
+            if _IsMessageMapField(field):
+              for key in field_value:
+                copy[key].MergeFrom(field_value[key])
+            else:
+              copy.update(field_value)
+          else:
+            for val in field_value:
+              if isinstance(val, dict):
+                copy.add(**val)
+              else:
+                copy.add().MergeFrom(val)
         else:  # Scalar
+          if field.cpp_type == _FieldDescriptor.CPPTYPE_ENUM:
+            field_value = [_GetIntegerEnumValue(field.enum_type, val)
+                           for val in field_value]
           copy.extend(field_value)
         self._fields[field] = copy
       elif field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE:
         copy = field._default_constructor(self)
+        new_val = field_value
+        if isinstance(field_value, dict):
+          new_val = field.message_type._concrete_class(**field_value)
         try:
-          copy.MergeFrom(field_value)
+          copy.MergeFrom(new_val)
         except TypeError:
           _ReraiseTypeErrorWithFieldName(message_descriptor.name, field_name)
         self._fields[field] = copy
       else:
+        if field.cpp_type == _FieldDescriptor.CPPTYPE_ENUM:
+          field_value = _GetIntegerEnumValue(field.enum_type, field_value)
         try:
           setattr(self, field_name, field_value)
         except TypeError:
@@ -758,6 +850,26 @@ def _AddHasExtensionMethod(cls):
       return extension_handle in self._fields
   cls.HasExtension = HasExtension
 
+def _UnpackAny(msg):
+  type_url = msg.type_url
+  db = symbol_database.Default()
+
+  if not type_url:
+    return None
+
+  # TODO(haberman): For now we just strip the hostname.  Better logic will be
+  # required.
+  type_name = type_url.split("/")[-1]
+  descriptor = db.pool.FindMessageTypeByName(type_name)
+
+  if descriptor is None:
+    return None
+
+  message_class = db.GetPrototype(descriptor)
+  message = message_class()
+
+  message.ParseFromString(msg.value)
+  return message
 
 def _AddEqualsMethod(message_descriptor, cls):
   """Helper for _AddMessageMethods()."""
@@ -769,6 +881,12 @@ def _AddEqualsMethod(message_descriptor, cls):
     if self is other:
       return True
 
+    if self.DESCRIPTOR.full_name == "google.protobuf.Any":
+      any_a = _UnpackAny(self)
+      any_b = _UnpackAny(other)
+      if any_a and any_b:
+        return any_a == any_b
+
     if not self.ListFields() == other.ListFields():
       return False
 
@@ -961,6 +1079,9 @@ def _AddIsInitializedMethod(message_descriptor, cls):
     for field, value in list(self._fields.items()):  # dict can change size!
       if field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE:
         if field.label == _FieldDescriptor.LABEL_REPEATED:
+          if (field.message_type.has_options and
+              field.message_type.GetOptions().map_entry):
+            continue
           for element in value:
             if not element.IsInitialized():
               if errors is not None:
@@ -996,16 +1117,26 @@ def _AddIsInitializedMethod(message_descriptor, cls):
         else:
           name = field.name
 
-        if field.label == _FieldDescriptor.LABEL_REPEATED:
+        if _IsMapField(field):
+          if _IsMessageMapField(field):
+            for key in value:
+              element = value[key]
+              prefix = "%s[%d]." % (name, key)
+              sub_errors = element.FindInitializationErrors()
+              errors += [prefix + error for error in sub_errors]
+          else:
+            # ScalarMaps can't have any initialization errors.
+            pass
+        elif field.label == _FieldDescriptor.LABEL_REPEATED:
           for i in xrange(len(value)):
             element = value[i]
             prefix = "%s[%d]." % (name, i)
             sub_errors = element.FindInitializationErrors()
-            errors += [ prefix + error for error in sub_errors ]
+            errors += [prefix + error for error in sub_errors]
         else:
           prefix = name + "."
           sub_errors = value.FindInitializationErrors()
-          errors += [ prefix + error for error in sub_errors ]
+          errors += [prefix + error for error in sub_errors]
 
     return errors