aboutsummaryrefslogtreecommitdiff
path: root/python/google/protobuf/pyext
diff options
context:
space:
mode:
authorBo Yang <teboring@google.com>2015-05-21 14:28:59 -0700
committerBo Yang <teboring@google.com>2015-05-21 19:32:02 -0700
commit5db217305f37a79eeccd70f000088a06ec82fcec (patch)
treebe53dcf0c0b47ef9178ab8a6fa5c1946ee84a28f /python/google/protobuf/pyext
parent56095026ccc2f755a6fdb296e30c3ddec8f556a2 (diff)
downloadprotobuf-5db217305f37a79eeccd70f000088a06ec82fcec.tar.gz
protobuf-5db217305f37a79eeccd70f000088a06ec82fcec.tar.bz2
protobuf-5db217305f37a79eeccd70f000088a06ec82fcec.zip
down-integrate internal changes
Diffstat (limited to 'python/google/protobuf/pyext')
-rw-r--r--python/google/protobuf/pyext/descriptor.cc287
-rw-r--r--python/google/protobuf/pyext/descriptor.h21
-rw-r--r--python/google/protobuf/pyext/descriptor_containers.cc26
-rw-r--r--python/google/protobuf/pyext/descriptor_containers.h5
-rw-r--r--python/google/protobuf/pyext/descriptor_pool.cc185
-rw-r--r--python/google/protobuf/pyext/descriptor_pool.h8
-rw-r--r--python/google/protobuf/pyext/extension_dict.cc17
-rw-r--r--python/google/protobuf/pyext/message.cc432
-rw-r--r--python/google/protobuf/pyext/message.h13
-rw-r--r--python/google/protobuf/pyext/message_map_container.cc540
-rw-r--r--python/google/protobuf/pyext/message_map_container.h117
-rw-r--r--python/google/protobuf/pyext/repeated_composite_container.cc56
-rw-r--r--python/google/protobuf/pyext/repeated_composite_container.h10
-rw-r--r--python/google/protobuf/pyext/repeated_scalar_container.cc8
-rw-r--r--python/google/protobuf/pyext/scalar_map_container.cc514
-rw-r--r--python/google/protobuf/pyext/scalar_map_container.h110
16 files changed, 1948 insertions, 401 deletions
diff --git a/python/google/protobuf/pyext/descriptor.cc b/python/google/protobuf/pyext/descriptor.cc
index e77d0bb9..2160757b 100644
--- a/python/google/protobuf/pyext/descriptor.cc
+++ b/python/google/protobuf/pyext/descriptor.cc
@@ -43,8 +43,6 @@
#include <google/protobuf/pyext/message.h>
#include <google/protobuf/pyext/scoped_pyobject_ptr.h>
-#define C(str) const_cast<char*>(str)
-
#if PY_MAJOR_VERSION >= 3
#define PyString_FromStringAndSize PyUnicode_FromStringAndSize
#define PyString_Check PyUnicode_Check
@@ -257,8 +255,14 @@ namespace descriptor {
// Creates or retrieve a Python descriptor of the specified type.
// Objects are interned: the same descriptor will return the same object if it
// was kept alive.
+// 'was_created' is an optional pointer to a bool, and is set to true if a new
+// object was allocated.
// Always return a new reference.
-PyObject* NewInternedDescriptor(PyTypeObject* type, const void* descriptor) {
+PyObject* NewInternedDescriptor(PyTypeObject* type, const void* descriptor,
+ bool* was_created) {
+ if (was_created) {
+ *was_created = false;
+ }
if (descriptor == NULL) {
PyErr_BadInternalCall();
return NULL;
@@ -283,6 +287,9 @@ PyObject* NewInternedDescriptor(PyTypeObject* type, const void* descriptor) {
GetDescriptorPool()->interned_descriptors->insert(
std::make_pair(descriptor, reinterpret_cast<PyObject*>(py_descriptor)));
+ if (was_created) {
+ *was_created = true;
+ }
return reinterpret_cast<PyObject*>(py_descriptor);
}
@@ -298,9 +305,7 @@ static PyGetSetDef Getters[] = {
PyTypeObject PyBaseDescriptor_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
- // Keep the fully qualified _message symbol in a line for opensource.
- "google.protobuf.internal._message."
- "DescriptorBase", // tp_name
+ FULL_MODULE_NAME ".DescriptorBase", // tp_name
sizeof(PyBaseDescriptor), // tp_basicsize
0, // tp_itemsize
(destructor)Dealloc, // tp_dealloc
@@ -357,7 +362,7 @@ static PyObject* GetFullName(PyBaseDescriptor* self, void *closure) {
}
static PyObject* GetFile(PyBaseDescriptor *self, void *closure) {
- return PyFileDescriptor_New(_GetDescriptor(self)->file());
+ return PyFileDescriptor_FromDescriptor(_GetDescriptor(self)->file());
}
static PyObject* GetConcreteClass(PyBaseDescriptor* self, void *closure) {
@@ -367,17 +372,6 @@ static PyObject* GetConcreteClass(PyBaseDescriptor* self, void *closure) {
return concrete_class;
}
-static int SetConcreteClass(PyBaseDescriptor *self, PyObject *value,
- void *closure) {
- // This attribute is also set from reflection.py. Check that it's actually a
- // no-op.
- if (value != cdescriptor_pool::GetMessageClass(
- GetDescriptorPool(), _GetDescriptor(self))) {
- PyErr_SetString(PyExc_AttributeError, "Cannot change _concrete_class");
- }
- return 0;
-}
-
static PyObject* GetFieldsByName(PyBaseDescriptor* self, void *closure) {
return NewMessageFieldsByName(_GetDescriptor(self));
}
@@ -452,7 +446,7 @@ static PyObject* GetContainingType(PyBaseDescriptor *self, void *closure) {
const Descriptor* containing_type =
_GetDescriptor(self)->containing_type();
if (containing_type) {
- return PyMessageDescriptor_New(containing_type);
+ return PyMessageDescriptor_FromDescriptor(containing_type);
} else {
Py_RETURN_NONE;
}
@@ -515,29 +509,34 @@ static PyObject* GetSyntax(PyBaseDescriptor *self, void *closure) {
}
static PyGetSetDef Getters[] = {
- { C("name"), (getter)GetName, NULL, "Last name", NULL},
- { C("full_name"), (getter)GetFullName, NULL, "Full name", NULL},
- { C("_concrete_class"), (getter)GetConcreteClass, (setter)SetConcreteClass, "concrete class", NULL},
- { C("file"), (getter)GetFile, NULL, "File descriptor", NULL},
-
- { C("fields"), (getter)GetFieldsSeq, NULL, "Fields sequence", NULL},
- { C("fields_by_name"), (getter)GetFieldsByName, NULL, "Fields by name", NULL},
- { C("fields_by_number"), (getter)GetFieldsByNumber, NULL, "Fields by number", NULL},
- { C("nested_types"), (getter)GetNestedTypesSeq, NULL, "Nested types sequence", NULL},
- { C("nested_types_by_name"), (getter)GetNestedTypesByName, NULL, "Nested types by name", NULL},
- { C("extensions"), (getter)GetExtensions, NULL, "Extensions Sequence", NULL},
- { C("extensions_by_name"), (getter)GetExtensionsByName, NULL, "Extensions by name", NULL},
- { C("extension_ranges"), (getter)GetExtensionRanges, NULL, "Extension ranges", NULL},
- { C("enum_types"), (getter)GetEnumsSeq, NULL, "Enum sequence", NULL},
- { C("enum_types_by_name"), (getter)GetEnumTypesByName, NULL, "Enum types by name", NULL},
- { C("enum_values_by_name"), (getter)GetEnumValuesByName, NULL, "Enum values by name", NULL},
- { C("oneofs_by_name"), (getter)GetOneofsByName, NULL, "Oneofs by name", NULL},
- { C("oneofs"), (getter)GetOneofsSeq, NULL, "Oneofs by name", NULL},
- { C("containing_type"), (getter)GetContainingType, (setter)SetContainingType, "Containing type", NULL},
- { C("is_extendable"), (getter)IsExtendable, (setter)NULL, NULL, NULL},
- { C("has_options"), (getter)GetHasOptions, (setter)SetHasOptions, "Has Options", NULL},
- { C("_options"), (getter)NULL, (setter)SetOptions, "Options", NULL},
- { C("syntax"), (getter)GetSyntax, (setter)NULL, "Syntax", NULL},
+ { "name", (getter)GetName, NULL, "Last name"},
+ { "full_name", (getter)GetFullName, NULL, "Full name"},
+ { "_concrete_class", (getter)GetConcreteClass, NULL, "concrete class"},
+ { "file", (getter)GetFile, NULL, "File descriptor"},
+
+ { "fields", (getter)GetFieldsSeq, NULL, "Fields sequence"},
+ { "fields_by_name", (getter)GetFieldsByName, NULL, "Fields by name"},
+ { "fields_by_number", (getter)GetFieldsByNumber, NULL, "Fields by number"},
+ { "nested_types", (getter)GetNestedTypesSeq, NULL, "Nested types sequence"},
+ { "nested_types_by_name", (getter)GetNestedTypesByName, NULL,
+ "Nested types by name"},
+ { "extensions", (getter)GetExtensions, NULL, "Extensions Sequence"},
+ { "extensions_by_name", (getter)GetExtensionsByName, NULL,
+ "Extensions by name"},
+ { "extension_ranges", (getter)GetExtensionRanges, NULL, "Extension ranges"},
+ { "enum_types", (getter)GetEnumsSeq, NULL, "Enum sequence"},
+ { "enum_types_by_name", (getter)GetEnumTypesByName, NULL,
+ "Enum types by name"},
+ { "enum_values_by_name", (getter)GetEnumValuesByName, NULL,
+ "Enum values by name"},
+ { "oneofs_by_name", (getter)GetOneofsByName, NULL, "Oneofs by name"},
+ { "oneofs", (getter)GetOneofsSeq, NULL, "Oneofs by name"},
+ { "containing_type", (getter)GetContainingType, (setter)SetContainingType,
+ "Containing type"},
+ { "is_extendable", (getter)IsExtendable, (setter)NULL},
+ { "has_options", (getter)GetHasOptions, (setter)SetHasOptions, "Has Options"},
+ { "_options", (getter)NULL, (setter)SetOptions, "Options"},
+ { "syntax", (getter)GetSyntax, (setter)NULL, "Syntax"},
{NULL}
};
@@ -552,9 +551,7 @@ static PyMethodDef Methods[] = {
PyTypeObject PyMessageDescriptor_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
- // Keep the fully qualified _message symbol in a line for opensource.
- C("google.protobuf.internal._message."
- "MessageDescriptor"), // tp_name
+ FULL_MODULE_NAME ".MessageDescriptor", // tp_name
sizeof(PyBaseDescriptor), // tp_basicsize
0, // tp_itemsize
0, // tp_dealloc
@@ -573,7 +570,7 @@ PyTypeObject PyMessageDescriptor_Type = {
0, // tp_setattro
0, // tp_as_buffer
Py_TPFLAGS_DEFAULT, // tp_flags
- C("A Message Descriptor"), // tp_doc
+ "A Message Descriptor", // tp_doc
0, // tp_traverse
0, // tp_clear
0, // tp_richcompare
@@ -586,10 +583,10 @@ PyTypeObject PyMessageDescriptor_Type = {
&descriptor::PyBaseDescriptor_Type, // tp_base
};
-PyObject* PyMessageDescriptor_New(
+PyObject* PyMessageDescriptor_FromDescriptor(
const Descriptor* message_descriptor) {
return descriptor::NewInternedDescriptor(
- &PyMessageDescriptor_Type, message_descriptor);
+ &PyMessageDescriptor_Type, message_descriptor, NULL);
}
const Descriptor* PyMessageDescriptor_AsDescriptor(PyObject* obj) {
@@ -715,7 +712,7 @@ static PyObject* GetCDescriptor(PyObject *self, void *closure) {
static PyObject *GetEnumType(PyBaseDescriptor *self, void *closure) {
const EnumDescriptor* enum_type = _GetDescriptor(self)->enum_type();
if (enum_type) {
- return PyEnumDescriptor_New(enum_type);
+ return PyEnumDescriptor_FromDescriptor(enum_type);
} else {
Py_RETURN_NONE;
}
@@ -728,7 +725,7 @@ static int SetEnumType(PyBaseDescriptor *self, PyObject *value, void *closure) {
static PyObject *GetMessageType(PyBaseDescriptor *self, void *closure) {
const Descriptor* message_type = _GetDescriptor(self)->message_type();
if (message_type) {
- return PyMessageDescriptor_New(message_type);
+ return PyMessageDescriptor_FromDescriptor(message_type);
} else {
Py_RETURN_NONE;
}
@@ -743,7 +740,7 @@ static PyObject* GetContainingType(PyBaseDescriptor *self, void *closure) {
const Descriptor* containing_type =
_GetDescriptor(self)->containing_type();
if (containing_type) {
- return PyMessageDescriptor_New(containing_type);
+ return PyMessageDescriptor_FromDescriptor(containing_type);
} else {
Py_RETURN_NONE;
}
@@ -758,7 +755,7 @@ static PyObject* GetExtensionScope(PyBaseDescriptor *self, void *closure) {
const Descriptor* extension_scope =
_GetDescriptor(self)->extension_scope();
if (extension_scope) {
- return PyMessageDescriptor_New(extension_scope);
+ return PyMessageDescriptor_FromDescriptor(extension_scope);
} else {
Py_RETURN_NONE;
}
@@ -768,7 +765,7 @@ static PyObject* GetContainingOneof(PyBaseDescriptor *self, void *closure) {
const OneofDescriptor* containing_oneof =
_GetDescriptor(self)->containing_oneof();
if (containing_oneof) {
- return PyOneofDescriptor_New(containing_oneof);
+ return PyOneofDescriptor_FromDescriptor(containing_oneof);
} else {
Py_RETURN_NONE;
}
@@ -803,26 +800,30 @@ static int SetOptions(PyBaseDescriptor *self, PyObject *value,
static PyGetSetDef Getters[] = {
- { C("full_name"), (getter)GetFullName, NULL, "Full name", NULL},
- { C("name"), (getter)GetName, NULL, "Unqualified name", NULL},
- { C("type"), (getter)GetType, NULL, "C++ Type", NULL},
- { C("cpp_type"), (getter)GetCppType, NULL, "C++ Type", NULL},
- { C("label"), (getter)GetLabel, NULL, "Label", NULL},
- { C("number"), (getter)GetNumber, NULL, "Number", NULL},
- { C("index"), (getter)GetIndex, NULL, "Index", NULL},
- { C("default_value"), (getter)GetDefaultValue, NULL, "Default Value", NULL},
- { C("has_default_value"), (getter)HasDefaultValue, NULL, NULL, NULL},
- { C("is_extension"), (getter)IsExtension, NULL, "ID", NULL},
- { C("id"), (getter)GetID, NULL, "ID", NULL},
- { C("_cdescriptor"), (getter)GetCDescriptor, NULL, "HAACK REMOVE ME", NULL},
-
- { C("message_type"), (getter)GetMessageType, (setter)SetMessageType, "Message type", NULL},
- { C("enum_type"), (getter)GetEnumType, (setter)SetEnumType, "Enum type", NULL},
- { C("containing_type"), (getter)GetContainingType, (setter)SetContainingType, "Containing type", NULL},
- { C("extension_scope"), (getter)GetExtensionScope, (setter)NULL, "Extension scope", NULL},
- { C("containing_oneof"), (getter)GetContainingOneof, (setter)SetContainingOneof, "Containing oneof", NULL},
- { C("has_options"), (getter)GetHasOptions, (setter)SetHasOptions, "Has Options", NULL},
- { C("_options"), (getter)NULL, (setter)SetOptions, "Options", NULL},
+ { "full_name", (getter)GetFullName, NULL, "Full name"},
+ { "name", (getter)GetName, NULL, "Unqualified name"},
+ { "type", (getter)GetType, NULL, "C++ Type"},
+ { "cpp_type", (getter)GetCppType, NULL, "C++ Type"},
+ { "label", (getter)GetLabel, NULL, "Label"},
+ { "number", (getter)GetNumber, NULL, "Number"},
+ { "index", (getter)GetIndex, NULL, "Index"},
+ { "default_value", (getter)GetDefaultValue, NULL, "Default Value"},
+ { "has_default_value", (getter)HasDefaultValue},
+ { "is_extension", (getter)IsExtension, NULL, "ID"},
+ { "id", (getter)GetID, NULL, "ID"},
+ { "_cdescriptor", (getter)GetCDescriptor, NULL, "HAACK REMOVE ME"},
+
+ { "message_type", (getter)GetMessageType, (setter)SetMessageType,
+ "Message type"},
+ { "enum_type", (getter)GetEnumType, (setter)SetEnumType, "Enum type"},
+ { "containing_type", (getter)GetContainingType, (setter)SetContainingType,
+ "Containing type"},
+ { "extension_scope", (getter)GetExtensionScope, (setter)NULL,
+ "Extension scope"},
+ { "containing_oneof", (getter)GetContainingOneof, (setter)SetContainingOneof,
+ "Containing oneof"},
+ { "has_options", (getter)GetHasOptions, (setter)SetHasOptions, "Has Options"},
+ { "_options", (getter)NULL, (setter)SetOptions, "Options"},
{NULL}
};
@@ -835,8 +836,7 @@ static PyMethodDef Methods[] = {
PyTypeObject PyFieldDescriptor_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
- C("google.protobuf.internal."
- "_message.FieldDescriptor"), // tp_name
+ FULL_MODULE_NAME ".FieldDescriptor", // tp_name
sizeof(PyBaseDescriptor), // tp_basicsize
0, // tp_itemsize
0, // tp_dealloc
@@ -855,7 +855,7 @@ PyTypeObject PyFieldDescriptor_Type = {
0, // tp_setattro
0, // tp_as_buffer
Py_TPFLAGS_DEFAULT, // tp_flags
- C("A Field Descriptor"), // tp_doc
+ "A Field Descriptor", // tp_doc
0, // tp_traverse
0, // tp_clear
0, // tp_richcompare
@@ -868,10 +868,10 @@ PyTypeObject PyFieldDescriptor_Type = {
&descriptor::PyBaseDescriptor_Type, // tp_base
};
-PyObject* PyFieldDescriptor_New(
+PyObject* PyFieldDescriptor_FromDescriptor(
const FieldDescriptor* field_descriptor) {
return descriptor::NewInternedDescriptor(
- &PyFieldDescriptor_Type, field_descriptor);
+ &PyFieldDescriptor_Type, field_descriptor, NULL);
}
const FieldDescriptor* PyFieldDescriptor_AsDescriptor(PyObject* obj) {
@@ -900,7 +900,7 @@ static PyObject* GetName(PyBaseDescriptor *self, void *closure) {
}
static PyObject* GetFile(PyBaseDescriptor *self, void *closure) {
- return PyFileDescriptor_New(_GetDescriptor(self)->file());
+ return PyFileDescriptor_FromDescriptor(_GetDescriptor(self)->file());
}
static PyObject* GetEnumvaluesByName(PyBaseDescriptor* self, void *closure) {
@@ -919,7 +919,7 @@ static PyObject* GetContainingType(PyBaseDescriptor *self, void *closure) {
const Descriptor* containing_type =
_GetDescriptor(self)->containing_type();
if (containing_type) {
- return PyMessageDescriptor_New(containing_type);
+ return PyMessageDescriptor_FromDescriptor(containing_type);
} else {
Py_RETURN_NONE;
}
@@ -964,16 +964,19 @@ static PyMethodDef Methods[] = {
};
static PyGetSetDef Getters[] = {
- { C("full_name"), (getter)GetFullName, NULL, "Full name", NULL},
- { C("name"), (getter)GetName, NULL, "last name", NULL},
- { C("file"), (getter)GetFile, NULL, "File descriptor", NULL},
- { C("values"), (getter)GetEnumvaluesSeq, NULL, "values", NULL},
- { C("values_by_name"), (getter)GetEnumvaluesByName, NULL, "Enumvalues by name", NULL},
- { C("values_by_number"), (getter)GetEnumvaluesByNumber, NULL, "Enumvalues by number", NULL},
-
- { C("containing_type"), (getter)GetContainingType, (setter)SetContainingType, "Containing type", NULL},
- { C("has_options"), (getter)GetHasOptions, (setter)SetHasOptions, "Has Options", NULL},
- { C("_options"), (getter)NULL, (setter)SetOptions, "Options", NULL},
+ { "full_name", (getter)GetFullName, NULL, "Full name"},
+ { "name", (getter)GetName, NULL, "last name"},
+ { "file", (getter)GetFile, NULL, "File descriptor"},
+ { "values", (getter)GetEnumvaluesSeq, NULL, "values"},
+ { "values_by_name", (getter)GetEnumvaluesByName, NULL,
+ "Enum values by name"},
+ { "values_by_number", (getter)GetEnumvaluesByNumber, NULL,
+ "Enum values by number"},
+
+ { "containing_type", (getter)GetContainingType, (setter)SetContainingType,
+ "Containing type"},
+ { "has_options", (getter)GetHasOptions, (setter)SetHasOptions, "Has Options"},
+ { "_options", (getter)NULL, (setter)SetOptions, "Options"},
{NULL}
};
@@ -981,9 +984,7 @@ static PyGetSetDef Getters[] = {
PyTypeObject PyEnumDescriptor_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
- // Keep the fully qualified _message symbol in a line for opensource.
- C("google.protobuf.internal._message."
- "EnumDescriptor"), // tp_name
+ FULL_MODULE_NAME ".EnumDescriptor", // tp_name
sizeof(PyBaseDescriptor), // tp_basicsize
0, // tp_itemsize
0, // tp_dealloc
@@ -1002,7 +1003,7 @@ PyTypeObject PyEnumDescriptor_Type = {
0, // tp_setattro
0, // tp_as_buffer
Py_TPFLAGS_DEFAULT, // tp_flags
- C("A Enum Descriptor"), // tp_doc
+ "A Enum Descriptor", // tp_doc
0, // tp_traverse
0, // tp_clear
0, // tp_richcompare
@@ -1015,10 +1016,10 @@ PyTypeObject PyEnumDescriptor_Type = {
&descriptor::PyBaseDescriptor_Type, // tp_base
};
-PyObject* PyEnumDescriptor_New(
+PyObject* PyEnumDescriptor_FromDescriptor(
const EnumDescriptor* enum_descriptor) {
return descriptor::NewInternedDescriptor(
- &PyEnumDescriptor_Type, enum_descriptor);
+ &PyEnumDescriptor_Type, enum_descriptor, NULL);
}
namespace enumvalue_descriptor {
@@ -1042,7 +1043,7 @@ static PyObject* GetIndex(PyBaseDescriptor *self, void *closure) {
}
static PyObject* GetType(PyBaseDescriptor *self, void *closure) {
- return PyEnumDescriptor_New(_GetDescriptor(self)->type());
+ return PyEnumDescriptor_FromDescriptor(_GetDescriptor(self)->type());
}
static PyObject* GetHasOptions(PyBaseDescriptor *self, void *closure) {
@@ -1069,13 +1070,13 @@ static int SetOptions(PyBaseDescriptor *self, PyObject *value,
static PyGetSetDef Getters[] = {
- { C("name"), (getter)GetName, NULL, "name", NULL},
- { C("number"), (getter)GetNumber, NULL, "number", NULL},
- { C("index"), (getter)GetIndex, NULL, "index", NULL},
- { C("type"), (getter)GetType, NULL, "index", NULL},
+ { "name", (getter)GetName, NULL, "name"},
+ { "number", (getter)GetNumber, NULL, "number"},
+ { "index", (getter)GetIndex, NULL, "index"},
+ { "type", (getter)GetType, NULL, "index"},
- { C("has_options"), (getter)GetHasOptions, (setter)SetHasOptions, "Has Options", NULL},
- { C("_options"), (getter)NULL, (setter)SetOptions, "Options", NULL},
+ { "has_options", (getter)GetHasOptions, (setter)SetHasOptions, "Has Options"},
+ { "_options", (getter)NULL, (setter)SetOptions, "Options"},
{NULL}
};
@@ -1088,8 +1089,7 @@ static PyMethodDef Methods[] = {
PyTypeObject PyEnumValueDescriptor_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
- C("google.protobuf.internal."
- "_message.EnumValueDescriptor"), // tp_name
+ FULL_MODULE_NAME ".EnumValueDescriptor", // tp_name
sizeof(PyBaseDescriptor), // tp_basicsize
0, // tp_itemsize
0, // tp_dealloc
@@ -1108,7 +1108,7 @@ PyTypeObject PyEnumValueDescriptor_Type = {
0, // tp_setattro
0, // tp_as_buffer
Py_TPFLAGS_DEFAULT, // tp_flags
- C("A EnumValue Descriptor"), // tp_doc
+ "A EnumValue Descriptor", // tp_doc
0, // tp_traverse
0, // tp_clear
0, // tp_richcompare
@@ -1121,10 +1121,10 @@ PyTypeObject PyEnumValueDescriptor_Type = {
&descriptor::PyBaseDescriptor_Type, // tp_base
};
-PyObject* PyEnumValueDescriptor_New(
+PyObject* PyEnumValueDescriptor_FromDescriptor(
const EnumValueDescriptor* enumvalue_descriptor) {
return descriptor::NewInternedDescriptor(
- &PyEnumValueDescriptor_Type, enumvalue_descriptor);
+ &PyEnumValueDescriptor_Type, enumvalue_descriptor, NULL);
}
namespace file_descriptor {
@@ -1218,18 +1218,20 @@ static PyObject* CopyToProto(PyFileDescriptor *self, PyObject *target) {
}
static PyGetSetDef Getters[] = {
- { C("name"), (getter)GetName, NULL, "name", NULL},
- { C("package"), (getter)GetPackage, NULL, "package", NULL},
- { C("serialized_pb"), (getter)GetSerializedPb, NULL, NULL, NULL},
- { C("message_types_by_name"), (getter)GetMessageTypesByName, NULL, "Messages by name", NULL},
- { C("enum_types_by_name"), (getter)GetEnumTypesByName, NULL, "Enums by name", NULL},
- { C("extensions_by_name"), (getter)GetExtensionsByName, NULL, "Extensions by name", NULL},
- { C("dependencies"), (getter)GetDependencies, NULL, "Dependencies", NULL},
- { C("public_dependencies"), (getter)GetPublicDependencies, NULL, "Dependencies", NULL},
-
- { C("has_options"), (getter)GetHasOptions, (setter)SetHasOptions, "Has Options", NULL},
- { C("_options"), (getter)NULL, (setter)SetOptions, "Options", NULL},
- { C("syntax"), (getter)GetSyntax, (setter)NULL, "Syntax", NULL},
+ { "name", (getter)GetName, NULL, "name"},
+ { "package", (getter)GetPackage, NULL, "package"},
+ { "serialized_pb", (getter)GetSerializedPb},
+ { "message_types_by_name", (getter)GetMessageTypesByName, NULL,
+ "Messages by name"},
+ { "enum_types_by_name", (getter)GetEnumTypesByName, NULL, "Enums by name"},
+ { "extensions_by_name", (getter)GetExtensionsByName, NULL,
+ "Extensions by name"},
+ { "dependencies", (getter)GetDependencies, NULL, "Dependencies"},
+ { "public_dependencies", (getter)GetPublicDependencies, NULL, "Dependencies"},
+
+ { "has_options", (getter)GetHasOptions, (setter)SetHasOptions, "Has Options"},
+ { "_options", (getter)NULL, (setter)SetOptions, "Options"},
+ { "syntax", (getter)GetSyntax, (setter)NULL, "Syntax"},
{NULL}
};
@@ -1243,11 +1245,10 @@ static PyMethodDef Methods[] = {
PyTypeObject PyFileDescriptor_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
- C("google.protobuf.internal."
- "_message.FileDescriptor"), // tp_name
+ FULL_MODULE_NAME ".FileDescriptor", // tp_name
sizeof(PyFileDescriptor), // tp_basicsize
0, // tp_itemsize
- (destructor)file_descriptor::Dealloc, // tp_dealloc
+ (destructor)file_descriptor::Dealloc, // tp_dealloc
0, // tp_print
0, // tp_getattr
0, // tp_setattr
@@ -1263,7 +1264,7 @@ PyTypeObject PyFileDescriptor_Type = {
0, // tp_setattro
0, // tp_as_buffer
Py_TPFLAGS_DEFAULT, // tp_flags
- C("A File Descriptor"), // tp_doc
+ "A File Descriptor", // tp_doc
0, // tp_traverse
0, // tp_clear
0, // tp_richcompare
@@ -1284,23 +1285,28 @@ PyTypeObject PyFileDescriptor_Type = {
PyObject_Del, // tp_free
};
-PyObject* PyFileDescriptor_New(const FileDescriptor* file_descriptor) {
- return descriptor::NewInternedDescriptor(
- &PyFileDescriptor_Type, file_descriptor);
+PyObject* PyFileDescriptor_FromDescriptor(
+ const FileDescriptor* file_descriptor) {
+ return PyFileDescriptor_FromDescriptorWithSerializedPb(file_descriptor,
+ NULL);
}
-PyObject* PyFileDescriptor_NewWithPb(
+PyObject* PyFileDescriptor_FromDescriptorWithSerializedPb(
const FileDescriptor* file_descriptor, PyObject *serialized_pb) {
- PyObject* py_descriptor = PyFileDescriptor_New(file_descriptor);
+ bool was_created;
+ PyObject* py_descriptor = descriptor::NewInternedDescriptor(
+ &PyFileDescriptor_Type, file_descriptor, &was_created);
if (py_descriptor == NULL) {
return NULL;
}
- if (serialized_pb != NULL) {
+ if (was_created) {
PyFileDescriptor* cfile_descriptor =
reinterpret_cast<PyFileDescriptor*>(py_descriptor);
Py_XINCREF(serialized_pb);
cfile_descriptor->serialized_pb = serialized_pb;
}
+ // TODO(amauryfa): In the case of a cached object, check that serialized_pb
+ // is the same as before.
return py_descriptor;
}
@@ -1333,19 +1339,19 @@ static PyObject* GetContainingType(PyBaseDescriptor *self, void *closure) {
const Descriptor* containing_type =
_GetDescriptor(self)->containing_type();
if (containing_type) {
- return PyMessageDescriptor_New(containing_type);
+ return PyMessageDescriptor_FromDescriptor(containing_type);
} else {
Py_RETURN_NONE;
}
}
static PyGetSetDef Getters[] = {
- { C("name"), (getter)GetName, NULL, "Name", NULL},
- { C("full_name"), (getter)GetFullName, NULL, "Full name", NULL},
- { C("index"), (getter)GetIndex, NULL, "Index", NULL},
+ { "name", (getter)GetName, NULL, "Name"},
+ { "full_name", (getter)GetFullName, NULL, "Full name"},
+ { "index", (getter)GetIndex, NULL, "Index"},
- { C("containing_type"), (getter)GetContainingType, NULL, "Containing type", NULL},
- { C("fields"), (getter)GetFields, NULL, "Fields", NULL},
+ { "containing_type", (getter)GetContainingType, NULL, "Containing type"},
+ { "fields", (getter)GetFields, NULL, "Fields"},
{NULL}
};
@@ -1353,8 +1359,7 @@ static PyGetSetDef Getters[] = {
PyTypeObject PyOneofDescriptor_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
- C("google.protobuf.internal."
- "_message.OneofDescriptor"), // tp_name
+ FULL_MODULE_NAME ".OneofDescriptor", // tp_name
sizeof(PyBaseDescriptor), // tp_basicsize
0, // tp_itemsize
0, // tp_dealloc
@@ -1373,7 +1378,7 @@ PyTypeObject PyOneofDescriptor_Type = {
0, // tp_setattro
0, // tp_as_buffer
Py_TPFLAGS_DEFAULT, // tp_flags
- C("A Oneof Descriptor"), // tp_doc
+ "A Oneof Descriptor", // tp_doc
0, // tp_traverse
0, // tp_clear
0, // tp_richcompare
@@ -1386,10 +1391,10 @@ PyTypeObject PyOneofDescriptor_Type = {
&descriptor::PyBaseDescriptor_Type, // tp_base
};
-PyObject* PyOneofDescriptor_New(
+PyObject* PyOneofDescriptor_FromDescriptor(
const OneofDescriptor* oneof_descriptor) {
return descriptor::NewInternedDescriptor(
- &PyOneofDescriptor_Type, oneof_descriptor);
+ &PyOneofDescriptor_Type, oneof_descriptor, NULL);
}
// Add a enum values to a type dictionary.
diff --git a/python/google/protobuf/pyext/descriptor.h b/python/google/protobuf/pyext/descriptor.h
index ba6e7298..b2550406 100644
--- a/python/google/protobuf/pyext/descriptor.h
+++ b/python/google/protobuf/pyext/descriptor.h
@@ -48,21 +48,24 @@ extern PyTypeObject PyEnumValueDescriptor_Type;
extern PyTypeObject PyFileDescriptor_Type;
extern PyTypeObject PyOneofDescriptor_Type;
-// Return a new reference to a Descriptor object.
+// Wraps a Descriptor in a Python object.
// The C++ pointer is usually borrowed from the global DescriptorPool.
// In any case, it must stay alive as long as the Python object.
-PyObject* PyMessageDescriptor_New(const Descriptor* descriptor);
-PyObject* PyFieldDescriptor_New(const FieldDescriptor* descriptor);
-PyObject* PyEnumDescriptor_New(const EnumDescriptor* descriptor);
-PyObject* PyEnumValueDescriptor_New(const EnumValueDescriptor* descriptor);
-PyObject* PyOneofDescriptor_New(const OneofDescriptor* descriptor);
-PyObject* PyFileDescriptor_New(const FileDescriptor* file_descriptor);
+// Returns a new reference.
+PyObject* PyMessageDescriptor_FromDescriptor(const Descriptor* descriptor);
+PyObject* PyFieldDescriptor_FromDescriptor(const FieldDescriptor* descriptor);
+PyObject* PyEnumDescriptor_FromDescriptor(const EnumDescriptor* descriptor);
+PyObject* PyEnumValueDescriptor_FromDescriptor(
+ const EnumValueDescriptor* descriptor);
+PyObject* PyOneofDescriptor_FromDescriptor(const OneofDescriptor* descriptor);
+PyObject* PyFileDescriptor_FromDescriptor(
+ const FileDescriptor* file_descriptor);
// Alternate constructor of PyFileDescriptor, used when we already have a
// serialized FileDescriptorProto that can be cached.
// Returns a new reference.
-PyObject* PyFileDescriptor_NewWithPb(const FileDescriptor* file_descriptor,
- PyObject* serialized_pb);
+PyObject* PyFileDescriptor_FromDescriptorWithSerializedPb(
+ const FileDescriptor* file_descriptor, PyObject* serialized_pb);
// Return the C++ descriptor pointer.
// This function checks the parameter type; on error, return NULL with a Python
diff --git a/python/google/protobuf/pyext/descriptor_containers.cc b/python/google/protobuf/pyext/descriptor_containers.cc
index 06edebf8..92e11e31 100644
--- a/python/google/protobuf/pyext/descriptor_containers.cc
+++ b/python/google/protobuf/pyext/descriptor_containers.cc
@@ -898,7 +898,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) {
}
static PyObject* NewObjectFromItem(ItemDescriptor item) {
- return PyFieldDescriptor_New(item);
+ return PyFieldDescriptor_FromDescriptor(item);
}
static const string& GetItemName(ItemDescriptor item) {
@@ -956,7 +956,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) {
}
static PyObject* NewObjectFromItem(ItemDescriptor item) {
- return PyMessageDescriptor_New(item);
+ return PyMessageDescriptor_FromDescriptor(item);
}
static const string& GetItemName(ItemDescriptor item) {
@@ -1006,7 +1006,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) {
}
static PyObject* NewObjectFromItem(ItemDescriptor item) {
- return PyEnumDescriptor_New(item);
+ return PyEnumDescriptor_FromDescriptor(item);
}
static const string& GetItemName(ItemDescriptor item) {
@@ -1082,7 +1082,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) {
}
static PyObject* NewObjectFromItem(ItemDescriptor item) {
- return PyEnumValueDescriptor_New(item);
+ return PyEnumValueDescriptor_FromDescriptor(item);
}
static const string& GetItemName(ItemDescriptor item) {
@@ -1124,7 +1124,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) {
}
static PyObject* NewObjectFromItem(ItemDescriptor item) {
- return PyFieldDescriptor_New(item);
+ return PyFieldDescriptor_FromDescriptor(item);
}
static const string& GetItemName(ItemDescriptor item) {
@@ -1174,7 +1174,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) {
}
static PyObject* NewObjectFromItem(ItemDescriptor item) {
- return PyOneofDescriptor_New(item);
+ return PyOneofDescriptor_FromDescriptor(item);
}
static const string& GetItemName(ItemDescriptor item) {
@@ -1238,7 +1238,7 @@ static ItemDescriptor GetByNumber(PyContainer* self, int number) {
}
static PyObject* NewObjectFromItem(ItemDescriptor item) {
- return PyEnumValueDescriptor_New(item);
+ return PyEnumValueDescriptor_FromDescriptor(item);
}
static const string& GetItemName(ItemDescriptor item) {
@@ -1302,7 +1302,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) {
}
static PyObject* NewObjectFromItem(ItemDescriptor item) {
- return PyFieldDescriptor_New(item);
+ return PyFieldDescriptor_FromDescriptor(item);
}
static int GetItemIndex(ItemDescriptor item) {
@@ -1354,7 +1354,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) {
}
static PyObject* NewObjectFromItem(ItemDescriptor item) {
- return PyMessageDescriptor_New(item);
+ return PyMessageDescriptor_FromDescriptor(item);
}
static const string& GetItemName(ItemDescriptor item) {
@@ -1400,7 +1400,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) {
}
static PyObject* NewObjectFromItem(ItemDescriptor item) {
- return PyEnumDescriptor_New(item);
+ return PyEnumDescriptor_FromDescriptor(item);
}
static const string& GetItemName(ItemDescriptor item) {
@@ -1446,7 +1446,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) {
}
static PyObject* NewObjectFromItem(ItemDescriptor item) {
- return PyFieldDescriptor_New(item);
+ return PyFieldDescriptor_FromDescriptor(item);
}
static const string& GetItemName(ItemDescriptor item) {
@@ -1488,7 +1488,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) {
}
static PyObject* NewObjectFromItem(ItemDescriptor item) {
- return PyFileDescriptor_New(item);
+ return PyFileDescriptor_FromDescriptor(item);
}
static DescriptorContainerDef ContainerDef = {
@@ -1522,7 +1522,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) {
}
static PyObject* NewObjectFromItem(ItemDescriptor item) {
- return PyFileDescriptor_New(item);
+ return PyFileDescriptor_FromDescriptor(item);
}
static DescriptorContainerDef ContainerDef = {
diff --git a/python/google/protobuf/pyext/descriptor_containers.h b/python/google/protobuf/pyext/descriptor_containers.h
index d81537de..8fbdaff9 100644
--- a/python/google/protobuf/pyext/descriptor_containers.h
+++ b/python/google/protobuf/pyext/descriptor_containers.h
@@ -28,6 +28,9 @@
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_CONTAINERS_H__
+#define GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_CONTAINERS_H__
+
// Mappings and Sequences of descriptors.
// They implement containers like fields_by_name, EnumDescriptor.values...
// See descriptor_containers.cc for more description.
@@ -92,4 +95,6 @@ PyObject* NewFilePublicDependencies(const FileDescriptor* descriptor);
} // namespace python
} // namespace protobuf
+
} // namespace google
+#endif // GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_CONTAINERS_H__
diff --git a/python/google/protobuf/pyext/descriptor_pool.cc b/python/google/protobuf/pyext/descriptor_pool.cc
index bc3077bc..ecd90847 100644
--- a/python/google/protobuf/pyext/descriptor_pool.cc
+++ b/python/google/protobuf/pyext/descriptor_pool.cc
@@ -35,10 +35,9 @@
#include <google/protobuf/descriptor.pb.h>
#include <google/protobuf/pyext/descriptor_pool.h>
#include <google/protobuf/pyext/descriptor.h>
+#include <google/protobuf/pyext/message.h>
#include <google/protobuf/pyext/scoped_pyobject_ptr.h>
-#define C(str) const_cast<char*>(str)
-
#if PY_MAJOR_VERSION >= 3
#define PyString_FromStringAndSize PyUnicode_FromStringAndSize
#if PY_VERSION_HEX < 0x03030000
@@ -108,11 +107,11 @@ PyObject* FindMessageByName(PyDescriptorPool* self, PyObject* arg) {
self->pool->FindMessageTypeByName(string(name, name_size));
if (message_descriptor == NULL) {
- PyErr_Format(PyExc_TypeError, "Couldn't find message %.200s", name);
+ PyErr_Format(PyExc_KeyError, "Couldn't find message %.200s", name);
return NULL;
}
- return PyMessageDescriptor_New(message_descriptor);
+ return PyMessageDescriptor_FromDescriptor(message_descriptor);
}
// Add a message class to our database.
@@ -158,6 +157,24 @@ PyObject *GetMessageClass(PyDescriptorPool* self,
}
}
+PyObject* FindFileByName(PyDescriptorPool* self, PyObject* arg) {
+ Py_ssize_t name_size;
+ char* name;
+ if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
+ return NULL;
+ }
+
+ const FileDescriptor* file_descriptor =
+ self->pool->FindFileByName(string(name, name_size));
+ if (file_descriptor == NULL) {
+ PyErr_Format(PyExc_KeyError, "Couldn't find file %.200s",
+ name);
+ return NULL;
+ }
+
+ return PyFileDescriptor_FromDescriptor(file_descriptor);
+}
+
PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* arg) {
Py_ssize_t name_size;
char* name;
@@ -168,12 +185,12 @@ PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* arg) {
const FieldDescriptor* field_descriptor =
self->pool->FindFieldByName(string(name, name_size));
if (field_descriptor == NULL) {
- PyErr_Format(PyExc_TypeError, "Couldn't find field %.200s",
+ PyErr_Format(PyExc_KeyError, "Couldn't find field %.200s",
name);
return NULL;
}
- return PyFieldDescriptor_New(field_descriptor);
+ return PyFieldDescriptor_FromDescriptor(field_descriptor);
}
PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg) {
@@ -186,11 +203,11 @@ PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg) {
const FieldDescriptor* field_descriptor =
self->pool->FindExtensionByName(string(name, name_size));
if (field_descriptor == NULL) {
- PyErr_Format(PyExc_TypeError, "Couldn't find field %.200s", name);
+ PyErr_Format(PyExc_KeyError, "Couldn't find extension field %.200s", name);
return NULL;
}
- return PyFieldDescriptor_New(field_descriptor);
+ return PyFieldDescriptor_FromDescriptor(field_descriptor);
}
PyObject* FindEnumTypeByName(PyDescriptorPool* self, PyObject* arg) {
@@ -203,11 +220,11 @@ PyObject* FindEnumTypeByName(PyDescriptorPool* self, PyObject* arg) {
const EnumDescriptor* enum_descriptor =
self->pool->FindEnumTypeByName(string(name, name_size));
if (enum_descriptor == NULL) {
- PyErr_Format(PyExc_TypeError, "Couldn't find enum %.200s", name);
+ PyErr_Format(PyExc_KeyError, "Couldn't find enum %.200s", name);
return NULL;
}
- return PyEnumDescriptor_New(enum_descriptor);
+ return PyEnumDescriptor_FromDescriptor(enum_descriptor);
}
PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg) {
@@ -220,70 +237,13 @@ PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg) {
const OneofDescriptor* oneof_descriptor =
self->pool->FindOneofByName(string(name, name_size));
if (oneof_descriptor == NULL) {
- PyErr_Format(PyExc_TypeError, "Couldn't find oneof %.200s", name);
+ PyErr_Format(PyExc_KeyError, "Couldn't find oneof %.200s", name);
return NULL;
}
- return PyOneofDescriptor_New(oneof_descriptor);
+ return PyOneofDescriptor_FromDescriptor(oneof_descriptor);
}
-static PyMethodDef Methods[] = {
- { C("FindFieldByName"),
- (PyCFunction)FindFieldByName,
- METH_O,
- C("Searches for a field descriptor by full name.") },
- { C("FindExtensionByName"),
- (PyCFunction)FindExtensionByName,
- METH_O,
- C("Searches for extension descriptor by full name.") },
- {NULL}
-};
-
-} // namespace cdescriptor_pool
-
-PyTypeObject PyDescriptorPool_Type = {
- PyVarObject_HEAD_INIT(&PyType_Type, 0)
- C("google.protobuf.internal."
- "_message.DescriptorPool"), // tp_name
- sizeof(PyDescriptorPool), // tp_basicsize
- 0, // tp_itemsize
- (destructor)cdescriptor_pool::Dealloc, // tp_dealloc
- 0, // tp_print
- 0, // tp_getattr
- 0, // tp_setattr
- 0, // tp_compare
- 0, // tp_repr
- 0, // tp_as_number
- 0, // tp_as_sequence
- 0, // tp_as_mapping
- 0, // tp_hash
- 0, // tp_call
- 0, // tp_str
- 0, // tp_getattro
- 0, // tp_setattro
- 0, // tp_as_buffer
- Py_TPFLAGS_DEFAULT, // tp_flags
- C("A Descriptor Pool"), // tp_doc
- 0, // tp_traverse
- 0, // tp_clear
- 0, // tp_richcompare
- 0, // tp_weaklistoffset
- 0, // tp_iter
- 0, // tp_iternext
- cdescriptor_pool::Methods, // tp_methods
- 0, // tp_members
- 0, // tp_getset
- 0, // tp_base
- 0, // tp_dict
- 0, // tp_descr_get
- 0, // tp_descr_set
- 0, // tp_dictoffset
- 0, // tp_init
- 0, // tp_alloc
- 0, // tp_new
- PyObject_Del, // tp_free
-};
-
// The code below loads new Descriptors from a serialized FileDescriptorProto.
@@ -301,6 +261,7 @@ class BuildFileErrorCollector : public DescriptorPool::ErrorCollector {
if (!had_errors) {
error_message +=
("Invalid proto descriptor for file \"" + filename + "\":\n");
+ had_errors = true;
}
// As this only happens on failure and will result in the program not
// running at all, no effort is made to optimize this string manipulation.
@@ -311,7 +272,7 @@ class BuildFileErrorCollector : public DescriptorPool::ErrorCollector {
bool had_errors;
};
-PyObject* Python_BuildFile(PyObject* ignored, PyObject* serialized_pb) {
+PyObject* AddSerializedFile(PyDescriptorPool* self, PyObject* serialized_pb) {
char* message_type;
Py_ssize_t message_len;
@@ -330,13 +291,14 @@ PyObject* Python_BuildFile(PyObject* ignored, PyObject* serialized_pb) {
const FileDescriptor* generated_file =
DescriptorPool::generated_pool()->FindFileByName(file_proto.name());
if (generated_file != NULL) {
- return PyFileDescriptor_NewWithPb(generated_file, serialized_pb);
+ return PyFileDescriptor_FromDescriptorWithSerializedPb(
+ generated_file, serialized_pb);
}
BuildFileErrorCollector error_collector;
const FileDescriptor* descriptor =
- GetDescriptorPool()->pool->BuildFileCollectingErrors(file_proto,
- &error_collector);
+ self->pool->BuildFileCollectingErrors(file_proto,
+ &error_collector);
if (descriptor == NULL) {
PyErr_Format(PyExc_TypeError,
"Couldn't build proto file into descriptor pool!\n%s",
@@ -344,9 +306,84 @@ PyObject* Python_BuildFile(PyObject* ignored, PyObject* serialized_pb) {
return NULL;
}
- return PyFileDescriptor_NewWithPb(descriptor, serialized_pb);
+ return PyFileDescriptor_FromDescriptorWithSerializedPb(
+ descriptor, serialized_pb);
+}
+
+PyObject* Add(PyDescriptorPool* self, PyObject* file_descriptor_proto) {
+ ScopedPyObjectPtr serialized_pb(
+ PyObject_CallMethod(file_descriptor_proto, "SerializeToString", NULL));
+ if (serialized_pb == NULL) {
+ return NULL;
+ }
+ return AddSerializedFile(self, serialized_pb);
}
+static PyMethodDef Methods[] = {
+ { "Add", (PyCFunction)Add, METH_O,
+ "Adds the FileDescriptorProto and its types to this pool." },
+ { "AddSerializedFile", (PyCFunction)AddSerializedFile, METH_O,
+ "Adds a serialized FileDescriptorProto to this pool." },
+
+ { "FindFileByName", (PyCFunction)FindFileByName, METH_O,
+ "Searches for a file descriptor by its .proto name." },
+ { "FindMessageTypeByName", (PyCFunction)FindMessageByName, METH_O,
+ "Searches for a message descriptor by full name." },
+ { "FindFieldByName", (PyCFunction)FindFieldByName, METH_O,
+ "Searches for a field descriptor by full name." },
+ { "FindExtensionByName", (PyCFunction)FindExtensionByName, METH_O,
+ "Searches for extension descriptor by full name." },
+ { "FindEnumTypeByName", (PyCFunction)FindEnumTypeByName, METH_O,
+ "Searches for enum type descriptor by full name." },
+ { "FindOneofByName", (PyCFunction)FindOneofByName, METH_O,
+ "Searches for oneof descriptor by full name." },
+ {NULL}
+};
+
+} // namespace cdescriptor_pool
+
+PyTypeObject PyDescriptorPool_Type = {
+ PyVarObject_HEAD_INIT(&PyType_Type, 0)
+ FULL_MODULE_NAME ".DescriptorPool", // tp_name
+ sizeof(PyDescriptorPool), // tp_basicsize
+ 0, // tp_itemsize
+ (destructor)cdescriptor_pool::Dealloc, // tp_dealloc
+ 0, // tp_print
+ 0, // tp_getattr
+ 0, // tp_setattr
+ 0, // tp_compare
+ 0, // tp_repr
+ 0, // tp_as_number
+ 0, // tp_as_sequence
+ 0, // tp_as_mapping
+ 0, // tp_hash
+ 0, // tp_call
+ 0, // tp_str
+ 0, // tp_getattro
+ 0, // tp_setattro
+ 0, // tp_as_buffer
+ Py_TPFLAGS_DEFAULT, // tp_flags
+ "A Descriptor Pool", // tp_doc
+ 0, // tp_traverse
+ 0, // tp_clear
+ 0, // tp_richcompare
+ 0, // tp_weaklistoffset
+ 0, // tp_iter
+ 0, // tp_iternext
+ cdescriptor_pool::Methods, // tp_methods
+ 0, // tp_members
+ 0, // tp_getset
+ 0, // tp_base
+ 0, // tp_dict
+ 0, // tp_descr_get
+ 0, // tp_descr_set
+ 0, // tp_dictoffset
+ 0, // tp_init
+ 0, // tp_alloc
+ 0, // tp_new
+ PyObject_Del, // tp_free
+};
+
static PyDescriptorPool* global_cdescriptor_pool = NULL;
bool InitDescriptorPool() {
diff --git a/python/google/protobuf/pyext/descriptor_pool.h b/python/google/protobuf/pyext/descriptor_pool.h
index 4e494b89..efb1abeb 100644
--- a/python/google/protobuf/pyext/descriptor_pool.h
+++ b/python/google/protobuf/pyext/descriptor_pool.h
@@ -95,6 +95,8 @@ const Descriptor* FindMessageTypeByName(PyDescriptorPool* self,
const Descriptor* RegisterMessageClass(
PyDescriptorPool* self, PyObject* message_class, PyObject* descriptor);
+// The function below are also exposed as methods of the DescriptorPool type.
+
// Retrieves the Python class registered with the given message descriptor.
//
// Returns a *borrowed* reference if found, otherwise returns NULL with an
@@ -134,12 +136,8 @@ PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg);
} // namespace cdescriptor_pool
-// Implement the Python "_BuildFile" method, it takes a serialized
-// FileDescriptorProto, and adds it to the C++ DescriptorPool.
-// It returns a new FileDescriptor object, or NULL when an exception is raised.
-PyObject* Python_BuildFile(PyObject* ignored, PyObject* args);
-
// Retrieve the global descriptor pool owned by the _message module.
+// Returns a *borrowed* reference.
PyDescriptorPool* GetDescriptorPool();
// Initialize objects used by this module.
diff --git a/python/google/protobuf/pyext/extension_dict.cc b/python/google/protobuf/pyext/extension_dict.cc
index 8e38fc42..b8d18f8d 100644
--- a/python/google/protobuf/pyext/extension_dict.cc
+++ b/python/google/protobuf/pyext/extension_dict.cc
@@ -51,16 +51,6 @@ namespace python {
namespace extension_dict {
-// TODO(tibell): Always use self->message for clarity, just like in
-// RepeatedCompositeContainer.
-static Message* GetMessage(ExtensionDict* self) {
- if (self->parent != NULL) {
- return self->parent->message;
- } else {
- return self->message;
- }
-}
-
PyObject* len(ExtensionDict* self) {
#if PY_MAJOR_VERSION >= 3
return PyLong_FromLong(PyDict_Size(self->values));
@@ -89,7 +79,7 @@ int ReleaseExtension(ExtensionDict* self,
}
} else if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
if (cmessage::ReleaseSubMessage(
- GetMessage(self), descriptor,
+ self->parent, descriptor,
reinterpret_cast<CMessage*>(extension)) < 0) {
return -1;
}
@@ -109,7 +99,7 @@ PyObject* subscript(ExtensionDict* self, PyObject* key) {
if (descriptor->label() != FieldDescriptor::LABEL_REPEATED &&
descriptor->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) {
- return cmessage::InternalGetScalar(self->parent, descriptor);
+ return cmessage::InternalGetScalar(self->parent->message, descriptor);
}
PyObject* value = PyDict_GetItem(self->values, key);
@@ -266,8 +256,7 @@ static PyMethodDef Methods[] = {
PyTypeObject ExtensionDict_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
- "google.protobuf.internal."
- "cpp._message.ExtensionDict", // tp_name
+ FULL_MODULE_NAME ".ExtensionDict", // tp_name
sizeof(ExtensionDict), // tp_basicsize
0, // tp_itemsize
(destructor)extension_dict::dealloc, // tp_dealloc
diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc
index a2b357b2..a4843e8d 100644
--- a/python/google/protobuf/pyext/message.cc
+++ b/python/google/protobuf/pyext/message.cc
@@ -59,6 +59,8 @@
#include <google/protobuf/pyext/extension_dict.h>
#include <google/protobuf/pyext/repeated_composite_container.h>
#include <google/protobuf/pyext/repeated_scalar_container.h>
+#include <google/protobuf/pyext/message_map_container.h>
+#include <google/protobuf/pyext/scalar_map_container.h>
#include <google/protobuf/pyext/scoped_pyobject_ptr.h>
#include <google/protobuf/stubs/strutil.h>
@@ -93,9 +95,9 @@ static const FieldDescriptor* GetFieldDescriptor(
static const Descriptor* GetMessageDescriptor(PyTypeObject* cls);
static string GetMessageName(CMessage* self);
int InternalReleaseFieldByDescriptor(
+ CMessage* self,
const FieldDescriptor* field_descriptor,
- PyObject* composite_field,
- Message* parent_message);
+ PyObject* composite_field);
} // namespace cmessage
// ---------------------------------------------------------------------
@@ -127,10 +129,29 @@ static int VisitCompositeField(const FieldDescriptor* descriptor,
Visitor visitor) {
if (descriptor->label() == FieldDescriptor::LABEL_REPEATED) {
if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
- RepeatedCompositeContainer* container =
- reinterpret_cast<RepeatedCompositeContainer*>(child);
- if (visitor.VisitRepeatedCompositeContainer(container) == -1)
- return -1;
+ if (descriptor->is_map()) {
+ const Descriptor* entry_type = descriptor->message_type();
+ const FieldDescriptor* value_type =
+ entry_type->FindFieldByName("value");
+ if (value_type->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
+ MessageMapContainer* container =
+ reinterpret_cast<MessageMapContainer*>(child);
+ if (visitor.VisitMessageMapContainer(container) == -1) {
+ return -1;
+ }
+ } else {
+ ScalarMapContainer* container =
+ reinterpret_cast<ScalarMapContainer*>(child);
+ if (visitor.VisitScalarMapContainer(container) == -1) {
+ return -1;
+ }
+ }
+ } else {
+ RepeatedCompositeContainer* container =
+ reinterpret_cast<RepeatedCompositeContainer*>(child);
+ if (visitor.VisitRepeatedCompositeContainer(container) == -1)
+ return -1;
+ }
} else {
RepeatedScalarContainer* container =
reinterpret_cast<RepeatedScalarContainer*>(child);
@@ -444,7 +465,7 @@ static int MaybeReleaseOverlappingOneofField(
}
if (InternalReleaseFieldByDescriptor(
- existing_field, child_message, message) < 0) {
+ cmessage, existing_field, child_message) < 0) {
return -1;
}
return PyDict_DelItemString(cmessage->composite_fields, field_name);
@@ -483,6 +504,16 @@ struct FixupMessageReference : public ChildVisitor {
return 0;
}
+ int VisitScalarMapContainer(ScalarMapContainer* container) {
+ container->message = message_;
+ return 0;
+ }
+
+ int VisitMessageMapContainer(MessageMapContainer* container) {
+ container->message = message_;
+ return 0;
+ }
+
private:
Message* message_;
};
@@ -500,6 +531,9 @@ int AssureWritable(CMessage* self) {
self->message->GetDescriptor());
self->message = prototype->New();
self->owner.reset(self->message);
+ // Cascade the new owner to eventual children: even if this message is
+ // empty, some submessages or repeated containers might exist already.
+ SetOwner(self, self->owner);
} else {
// Otherwise, we need a mutable child message.
if (AssureWritable(self->parent) == -1)
@@ -520,8 +554,9 @@ int AssureWritable(CMessage* self) {
// When a CMessage is made writable its Message pointer is updated
// to point to a new mutable Message. When that happens we need to
// update any references to the old, read-only CMessage. There are
- // three places such references occur: RepeatedScalarContainer,
- // RepeatedCompositeContainer, and ExtensionDict.
+ // five places such references occur: RepeatedScalarContainer,
+ // RepeatedCompositeContainer, ScalarMapContainer, MessageMapContainer,
+ // and ExtensionDict.
if (self->extensions != NULL)
self->extensions->message = self->message;
if (ForEachCompositeField(self, FixupMessageReference(self->message)) == -1)
@@ -583,15 +618,43 @@ const FieldDescriptor* GetExtensionDescriptor(PyObject* extension) {
return PyFieldDescriptor_AsDescriptor(extension);
}
+// If value is a string, convert it into an enum value based on the labels in
+// descriptor, otherwise simply return value. Always returns a new reference.
+static PyObject* GetIntegerEnumValue(const FieldDescriptor& descriptor,
+ PyObject* value) {
+ if (PyString_Check(value) || PyUnicode_Check(value)) {
+ const EnumDescriptor* enum_descriptor = descriptor.enum_type();
+ if (enum_descriptor == NULL) {
+ PyErr_SetString(PyExc_TypeError, "not an enum field");
+ return NULL;
+ }
+ char* enum_label;
+ Py_ssize_t size;
+ if (PyString_AsStringAndSize(value, &enum_label, &size) < 0) {
+ return NULL;
+ }
+ const EnumValueDescriptor* enum_value_descriptor =
+ enum_descriptor->FindValueByName(string(enum_label, size));
+ if (enum_value_descriptor == NULL) {
+ PyErr_SetString(PyExc_ValueError, "unknown enum label");
+ return NULL;
+ }
+ return PyInt_FromLong(enum_value_descriptor->number());
+ }
+ Py_INCREF(value);
+ return value;
+}
+
// If cmessage_list is not NULL, this function releases values into the
// container CMessages instead of just removing. Repeated composite container
// needs to do this to make sure CMessages stay alive if they're still
// referenced after deletion. Repeated scalar container doesn't need to worry.
int InternalDeleteRepeatedField(
- Message* message,
+ CMessage* self,
const FieldDescriptor* field_descriptor,
PyObject* slice,
PyObject* cmessage_list) {
+ Message* message = self->message;
Py_ssize_t length, from, to, step, slice_length;
const Reflection* reflection = message->GetReflection();
int min, max;
@@ -665,7 +728,7 @@ int InternalDeleteRepeatedField(
CMessage* last_cmessage = reinterpret_cast<CMessage*>(
PyList_GET_ITEM(cmessage_list, PyList_GET_SIZE(cmessage_list) - 1));
repeated_composite_container::ReleaseLastTo(
- field_descriptor, message, last_cmessage);
+ self, field_descriptor, last_cmessage);
if (PySequence_DelItem(cmessage_list, -1) < 0) {
return -1;
}
@@ -696,16 +759,90 @@ int InitAttributes(CMessage* self, PyObject* kwargs) {
PyString_AsString(name));
return -1;
}
- if (descriptor->label() == FieldDescriptor::LABEL_REPEATED) {
+ if (descriptor->is_map()) {
+ ScopedPyObjectPtr map(GetAttr(self, name));
+ const FieldDescriptor* value_descriptor =
+ descriptor->message_type()->FindFieldByName("value");
+ if (value_descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
+ Py_ssize_t map_pos = 0;
+ PyObject* map_key;
+ PyObject* map_value;
+ while (PyDict_Next(value, &map_pos, &map_key, &map_value)) {
+ ScopedPyObjectPtr function_return;
+ function_return.reset(PyObject_GetItem(map.get(), map_key));
+ if (function_return.get() == NULL) {
+ return -1;
+ }
+ ScopedPyObjectPtr ok(PyObject_CallMethod(
+ function_return.get(), "MergeFrom", "O", map_value));
+ if (ok.get() == NULL) {
+ return -1;
+ }
+ }
+ } else {
+ ScopedPyObjectPtr function_return;
+ function_return.reset(
+ PyObject_CallMethod(map.get(), "update", "O", value));
+ if (function_return.get() == NULL) {
+ return -1;
+ }
+ }
+ } else if (descriptor->label() == FieldDescriptor::LABEL_REPEATED) {
ScopedPyObjectPtr container(GetAttr(self, name));
if (container == NULL) {
return -1;
}
if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
- if (repeated_composite_container::Extend(
- reinterpret_cast<RepeatedCompositeContainer*>(container.get()),
- value)
- == NULL) {
+ RepeatedCompositeContainer* rc_container =
+ reinterpret_cast<RepeatedCompositeContainer*>(container.get());
+ ScopedPyObjectPtr iter(PyObject_GetIter(value));
+ if (iter == NULL) {
+ PyErr_SetString(PyExc_TypeError, "Value must be iterable");
+ return -1;
+ }
+ ScopedPyObjectPtr next;
+ while ((next.reset(PyIter_Next(iter))) != NULL) {
+ PyObject* kwargs = (PyDict_Check(next) ? next.get() : NULL);
+ ScopedPyObjectPtr new_msg(
+ repeated_composite_container::Add(rc_container, NULL, kwargs));
+ if (new_msg == NULL) {
+ return -1;
+ }
+ if (kwargs == NULL) {
+ // next was not a dict, it's a message we need to merge
+ ScopedPyObjectPtr merged(
+ MergeFrom(reinterpret_cast<CMessage*>(new_msg.get()), next));
+ if (merged == NULL) {
+ return -1;
+ }
+ }
+ }
+ if (PyErr_Occurred()) {
+ // Check to see how PyIter_Next() exited.
+ return -1;
+ }
+ } else if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_ENUM) {
+ RepeatedScalarContainer* rs_container =
+ reinterpret_cast<RepeatedScalarContainer*>(container.get());
+ ScopedPyObjectPtr iter(PyObject_GetIter(value));
+ if (iter == NULL) {
+ PyErr_SetString(PyExc_TypeError, "Value must be iterable");
+ return -1;
+ }
+ ScopedPyObjectPtr next;
+ while ((next.reset(PyIter_Next(iter))) != NULL) {
+ ScopedPyObjectPtr enum_value(GetIntegerEnumValue(*descriptor, next));
+ if (enum_value == NULL) {
+ return -1;
+ }
+ ScopedPyObjectPtr new_msg(
+ repeated_scalar_container::Append(rs_container, enum_value));
+ if (new_msg == NULL) {
+ return -1;
+ }
+ }
+ if (PyErr_Occurred()) {
+ // Check to see how PyIter_Next() exited.
return -1;
}
} else {
@@ -721,12 +858,26 @@ int InitAttributes(CMessage* self, PyObject* kwargs) {
if (message == NULL) {
return -1;
}
- if (MergeFrom(reinterpret_cast<CMessage*>(message.get()),
- value) == NULL) {
- return -1;
+ CMessage* cmessage = reinterpret_cast<CMessage*>(message.get());
+ if (PyDict_Check(value)) {
+ if (InitAttributes(cmessage, value) < 0) {
+ return -1;
+ }
+ } else {
+ ScopedPyObjectPtr merged(MergeFrom(cmessage, value));
+ if (merged == NULL) {
+ return -1;
+ }
}
} else {
- if (SetAttr(self, name, value) < 0) {
+ ScopedPyObjectPtr new_val;
+ if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_ENUM) {
+ new_val.reset(GetIntegerEnumValue(*descriptor, value));
+ if (new_val == NULL) {
+ return -1;
+ }
+ }
+ if (SetAttr(self, name, (new_val == NULL) ? value : new_val) < 0) {
return -1;
}
}
@@ -789,7 +940,6 @@ static PyObject* New(PyTypeObject* type,
}
self->message = default_message->New();
self->owner.reset(self->message);
-
return reinterpret_cast<PyObject*>(self);
}
@@ -830,6 +980,16 @@ struct ClearWeakReferences : public ChildVisitor {
return 0;
}
+ int VisitScalarMapContainer(ScalarMapContainer* container) {
+ container->parent = NULL;
+ return 0;
+ }
+
+ int VisitMessageMapContainer(MessageMapContainer* container) {
+ container->parent = NULL;
+ return 0;
+ }
+
int VisitCMessage(CMessage* cmessage,
const FieldDescriptor* field_descriptor) {
cmessage->parent = NULL;
@@ -1064,6 +1224,16 @@ struct SetOwnerVisitor : public ChildVisitor {
return 0;
}
+ int VisitScalarMapContainer(ScalarMapContainer* container) {
+ scalar_map_container::SetOwner(container, new_owner_);
+ return 0;
+ }
+
+ int VisitMessageMapContainer(MessageMapContainer* container) {
+ message_map_container::SetOwner(container, new_owner_);
+ return 0;
+ }
+
int VisitCMessage(CMessage* cmessage,
const FieldDescriptor* field_descriptor) {
return SetOwner(cmessage, new_owner_);
@@ -1084,11 +1254,11 @@ int SetOwner(CMessage* self, const shared_ptr<Message>& new_owner) {
// Releases the message specified by 'field' and returns the
// pointer. If the field does not exist a new message is created using
// 'descriptor'. The caller takes ownership of the returned pointer.
-Message* ReleaseMessage(Message* message,
+Message* ReleaseMessage(CMessage* self,
const Descriptor* descriptor,
const FieldDescriptor* field_descriptor) {
- Message* released_message = message->GetReflection()->ReleaseMessage(
- message, field_descriptor, message_factory);
+ Message* released_message = self->message->GetReflection()->ReleaseMessage(
+ self->message, field_descriptor, message_factory);
// ReleaseMessage will return NULL which differs from
// child_cmessage->message, if the field does not exist. In this case,
// the latter points to the default instance via a const_cast<>, so we
@@ -1102,12 +1272,12 @@ Message* ReleaseMessage(Message* message,
return released_message;
}
-int ReleaseSubMessage(Message* message,
+int ReleaseSubMessage(CMessage* self,
const FieldDescriptor* field_descriptor,
CMessage* child_cmessage) {
// Release the Message
shared_ptr<Message> released_message(ReleaseMessage(
- message, child_cmessage->message->GetDescriptor(), field_descriptor));
+ self, child_cmessage->message->GetDescriptor(), field_descriptor));
child_cmessage->message = released_message.get();
child_cmessage->owner.swap(released_message);
child_cmessage->parent = NULL;
@@ -1119,8 +1289,8 @@ int ReleaseSubMessage(Message* message,
struct ReleaseChild : public ChildVisitor {
// message must outlive this object.
- explicit ReleaseChild(Message* parent_message) :
- parent_message_(parent_message) {}
+ explicit ReleaseChild(CMessage* parent) :
+ parent_(parent) {}
int VisitRepeatedCompositeContainer(RepeatedCompositeContainer* container) {
return repeated_composite_container::Release(
@@ -1132,23 +1302,33 @@ struct ReleaseChild : public ChildVisitor {
reinterpret_cast<RepeatedScalarContainer*>(container));
}
+ int VisitScalarMapContainer(ScalarMapContainer* container) {
+ return scalar_map_container::Release(
+ reinterpret_cast<ScalarMapContainer*>(container));
+ }
+
+ int VisitMessageMapContainer(MessageMapContainer* container) {
+ return message_map_container::Release(
+ reinterpret_cast<MessageMapContainer*>(container));
+ }
+
int VisitCMessage(CMessage* cmessage,
const FieldDescriptor* field_descriptor) {
- return ReleaseSubMessage(parent_message_, field_descriptor,
+ return ReleaseSubMessage(parent_, field_descriptor,
reinterpret_cast<CMessage*>(cmessage));
}
- Message* parent_message_;
+ CMessage* parent_;
};
int InternalReleaseFieldByDescriptor(
+ CMessage* self,
const FieldDescriptor* field_descriptor,
- PyObject* composite_field,
- Message* parent_message) {
+ PyObject* composite_field) {
return VisitCompositeField(
field_descriptor,
composite_field,
- ReleaseChild(parent_message));
+ ReleaseChild(self));
}
PyObject* ClearFieldByDescriptor(
@@ -1200,8 +1380,8 @@ PyObject* ClearField(CMessage* self, PyObject* arg) {
// Only release the field if there's a possibility that there are
// references to it.
if (composite_field != NULL) {
- if (InternalReleaseFieldByDescriptor(field_descriptor,
- composite_field, message) < 0) {
+ if (InternalReleaseFieldByDescriptor(self, field_descriptor,
+ composite_field) < 0) {
return NULL;
}
PyDict_DelItem(self->composite_fields, arg);
@@ -1219,7 +1399,7 @@ PyObject* ClearField(CMessage* self, PyObject* arg) {
PyObject* Clear(CMessage* self) {
AssureWritable(self);
- if (ForEachCompositeField(self, ReleaseChild(self->message)) == -1)
+ if (ForEachCompositeField(self, ReleaseChild(self)) == -1)
return NULL;
// The old ExtensionDict still aliases this CMessage, but all its
@@ -1582,7 +1762,8 @@ static PyObject* ListFields(CMessage* self) {
}
if (fields[i]->is_extension()) {
- ScopedPyObjectPtr extension_field(PyFieldDescriptor_New(fields[i]));
+ ScopedPyObjectPtr extension_field(
+ PyFieldDescriptor_FromDescriptor(fields[i]));
if (extension_field == NULL) {
return NULL;
}
@@ -1616,7 +1797,8 @@ static PyObject* ListFields(CMessage* self) {
PyErr_SetString(PyExc_ValueError, "bad string");
return NULL;
}
- ScopedPyObjectPtr field_descriptor(PyFieldDescriptor_New(fields[i]));
+ ScopedPyObjectPtr field_descriptor(
+ PyFieldDescriptor_FromDescriptor(fields[i]));
if (field_descriptor == NULL) {
return NULL;
}
@@ -1683,10 +1865,8 @@ static PyObject* RichCompare(CMessage* self, PyObject* other, int opid) {
}
}
-PyObject* InternalGetScalar(
- CMessage* self,
- const FieldDescriptor* field_descriptor) {
- Message* message = self->message;
+PyObject* InternalGetScalar(const Message* message,
+ const FieldDescriptor* field_descriptor) {
const Reflection* reflection = message->GetReflection();
if (!CheckFieldBelongsToMessage(field_descriptor, message)) {
@@ -1739,12 +1919,12 @@ PyObject* InternalGetScalar(
if (!message->GetReflection()->SupportsUnknownEnumValues() &&
!message->GetReflection()->HasField(*message, field_descriptor)) {
// Look for the value in the unknown fields.
- UnknownFieldSet* unknown_field_set =
- message->GetReflection()->MutableUnknownFields(message);
- for (int i = 0; i < unknown_field_set->field_count(); ++i) {
- if (unknown_field_set->field(i).number() ==
+ const UnknownFieldSet& unknown_field_set =
+ message->GetReflection()->GetUnknownFields(*message);
+ for (int i = 0; i < unknown_field_set.field_count(); ++i) {
+ if (unknown_field_set.field(i).number() ==
field_descriptor->number()) {
- result = PyInt_FromLong(unknown_field_set->field(i).varint());
+ result = PyInt_FromLong(unknown_field_set.field(i).varint());
break;
}
}
@@ -1793,21 +1973,16 @@ PyObject* InternalGetSubMessage(
return reinterpret_cast<PyObject*>(cmsg);
}
-int InternalSetScalar(
- CMessage* self,
+int InternalSetNonOneofScalar(
+ Message* message,
const FieldDescriptor* field_descriptor,
PyObject* arg) {
- Message* message = self->message;
const Reflection* reflection = message->GetReflection();
if (!CheckFieldBelongsToMessage(field_descriptor, message)) {
return -1;
}
- if (MaybeReleaseOverlappingOneofField(self, field_descriptor) < 0) {
- return -1;
- }
-
switch (field_descriptor->cpp_type()) {
case FieldDescriptor::CPPTYPE_INT32: {
GOOGLE_CHECK_GET_INT32(arg, value, -1);
@@ -1878,6 +2053,21 @@ int InternalSetScalar(
return 0;
}
+int InternalSetScalar(
+ CMessage* self,
+ const FieldDescriptor* field_descriptor,
+ PyObject* arg) {
+ if (!CheckFieldBelongsToMessage(field_descriptor, self->message)) {
+ return -1;
+ }
+
+ if (MaybeReleaseOverlappingOneofField(self, field_descriptor) < 0) {
+ return -1;
+ }
+
+ return InternalSetNonOneofScalar(self->message, field_descriptor, arg);
+}
+
PyObject* FromString(PyTypeObject* cls, PyObject* serialized) {
PyObject* py_cmsg = PyObject_CallObject(
reinterpret_cast<PyObject*>(cls), NULL);
@@ -1955,7 +2145,8 @@ static PyObject* AddDescriptors(PyObject* cls, PyObject* descriptor) {
// which was built previously.
for (int i = 0; i < message_descriptor->enum_type_count(); ++i) {
const EnumDescriptor* enum_descriptor = message_descriptor->enum_type(i);
- ScopedPyObjectPtr enum_type(PyEnumDescriptor_New(enum_descriptor));
+ ScopedPyObjectPtr enum_type(
+ PyEnumDescriptor_FromDescriptor(enum_descriptor));
if (enum_type == NULL) {
return NULL;
}
@@ -1993,7 +2184,7 @@ static PyObject* AddDescriptors(PyObject* cls, PyObject* descriptor) {
// which was defined previously.
for (int i = 0; i < message_descriptor->extension_count(); ++i) {
const google::protobuf::FieldDescriptor* field = message_descriptor->extension(i);
- ScopedPyObjectPtr extension_field(PyFieldDescriptor_New(field));
+ ScopedPyObjectPtr extension_field(PyFieldDescriptor_FromDescriptor(field));
if (extension_field == NULL) {
return NULL;
}
@@ -2097,26 +2288,6 @@ PyObject* SetState(CMessage* self, PyObject* state) {
}
// CMessage static methods:
-PyObject* _GetMessageDescriptor(PyObject* unused, PyObject* arg) {
- return cdescriptor_pool::FindMessageByName(GetDescriptorPool(), arg);
-}
-
-PyObject* _GetFieldDescriptor(PyObject* unused, PyObject* arg) {
- return cdescriptor_pool::FindFieldByName(GetDescriptorPool(), arg);
-}
-
-PyObject* _GetExtensionDescriptor(PyObject* unused, PyObject* arg) {
- return cdescriptor_pool::FindExtensionByName(GetDescriptorPool(), arg);
-}
-
-PyObject* _GetEnumDescriptor(PyObject* unused, PyObject* arg) {
- return cdescriptor_pool::FindEnumTypeByName(GetDescriptorPool(), arg);
-}
-
-PyObject* _GetOneofDescriptor(PyObject* unused, PyObject* arg) {
- return cdescriptor_pool::FindOneofByName(GetDescriptorPool(), arg);
-}
-
PyObject* _CheckCalledFromGeneratedFile(PyObject* unused,
PyObject* unused_arg) {
if (!_CalledFromGeneratedFile(1)) {
@@ -2188,21 +2359,6 @@ static PyMethodDef Methods[] = {
"or None if no field is set." },
// Static Methods.
- { "_BuildFile", (PyCFunction)Python_BuildFile, METH_O | METH_STATIC,
- "Registers a new protocol buffer file in the global C++ descriptor pool." },
- { "_GetMessageDescriptor", (PyCFunction)_GetMessageDescriptor,
- METH_O | METH_STATIC, "Finds a message descriptor in the message pool." },
- { "_GetFieldDescriptor", (PyCFunction)_GetFieldDescriptor,
- METH_O | METH_STATIC, "Finds a field descriptor in the message pool." },
- { "_GetExtensionDescriptor", (PyCFunction)_GetExtensionDescriptor,
- METH_O | METH_STATIC,
- "Finds a extension descriptor in the message pool." },
- { "_GetEnumDescriptor", (PyCFunction)_GetEnumDescriptor,
- METH_O | METH_STATIC,
- "Finds an enum descriptor in the message pool." },
- { "_GetOneofDescriptor", (PyCFunction)_GetOneofDescriptor,
- METH_O | METH_STATIC,
- "Finds an oneof descriptor in the message pool." },
{ "_CheckCalledFromGeneratedFile", (PyCFunction)_CheckCalledFromGeneratedFile,
METH_NOARGS | METH_STATIC,
"Raises TypeError if the caller is not in a _pb2.py file."},
@@ -2234,6 +2390,31 @@ PyObject* GetAttr(CMessage* self, PyObject* name) {
reinterpret_cast<PyObject*>(self), name);
}
+ if (field_descriptor->is_map()) {
+ PyObject* py_container = NULL;
+ const Descriptor* entry_type = field_descriptor->message_type();
+ const FieldDescriptor* value_type = entry_type->FindFieldByName("value");
+ if (value_type->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
+ PyObject* value_class = cdescriptor_pool::GetMessageClass(
+ GetDescriptorPool(), value_type->message_type());
+ if (value_class == NULL) {
+ return NULL;
+ }
+ py_container = message_map_container::NewContainer(self, field_descriptor,
+ value_class);
+ } else {
+ py_container = scalar_map_container::NewContainer(self, field_descriptor);
+ }
+ if (py_container == NULL) {
+ return NULL;
+ }
+ if (!SetCompositeField(self, name, py_container)) {
+ Py_DECREF(py_container);
+ return NULL;
+ }
+ return py_container;
+ }
+
if (field_descriptor->label() == FieldDescriptor::LABEL_REPEATED) {
PyObject* py_container = NULL;
if (field_descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
@@ -2267,7 +2448,7 @@ PyObject* GetAttr(CMessage* self, PyObject* name) {
return sub_message;
}
- return InternalGetScalar(self, field_descriptor);
+ return InternalGetScalar(self->message, field_descriptor);
}
int SetAttr(CMessage* self, PyObject* name, PyObject* value) {
@@ -2304,9 +2485,7 @@ int SetAttr(CMessage* self, PyObject* name, PyObject* value) {
PyTypeObject CMessage_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
- // Keep the fully qualified _message symbol in a line for opensource.
- "google.protobuf.pyext._message."
- "CMessage", // tp_name
+ FULL_MODULE_NAME ".CMessage", // tp_name
sizeof(CMessage), // tp_basicsize
0, // tp_itemsize
(destructor)cmessage::Dealloc, // tp_dealloc
@@ -2401,7 +2580,7 @@ void InitGlobals() {
k_extensions_by_name = PyString_FromString("_extensions_by_name");
k_extensions_by_number = PyString_FromString("_extensions_by_number");
- message_factory = new DynamicMessageFactory(GetDescriptorPool()->pool);
+ message_factory = new DynamicMessageFactory();
message_factory->SetDelegateToGeneratedFactory(true);
}
@@ -2469,6 +2648,61 @@ bool InitProto2MessageModule(PyObject *m) {
reinterpret_cast<PyObject*>(
&RepeatedCompositeContainer_Type));
+ // ScalarMapContainer_Type derives from our MutableMapping type.
+ PyObject* containers =
+ PyImport_ImportModule("google.protobuf.internal.containers");
+ if (containers == NULL) {
+ return false;
+ }
+
+ PyObject* mutable_mapping =
+ PyObject_GetAttrString(containers, "MutableMapping");
+ Py_DECREF(containers);
+
+ if (mutable_mapping == NULL) {
+ return false;
+ }
+
+ if (!PyObject_TypeCheck(mutable_mapping, &PyType_Type)) {
+ Py_DECREF(mutable_mapping);
+ return false;
+ }
+
+ ScalarMapContainer_Type.tp_base =
+ reinterpret_cast<PyTypeObject*>(mutable_mapping);
+
+ if (PyType_Ready(&ScalarMapContainer_Type) < 0) {
+ return false;
+ }
+
+ PyModule_AddObject(m, "ScalarMapContainer",
+ reinterpret_cast<PyObject*>(&ScalarMapContainer_Type));
+
+ if (PyType_Ready(&ScalarMapIterator_Type) < 0) {
+ return false;
+ }
+
+ PyModule_AddObject(m, "ScalarMapIterator",
+ reinterpret_cast<PyObject*>(&ScalarMapIterator_Type));
+
+ Py_INCREF(mutable_mapping);
+ MessageMapContainer_Type.tp_base =
+ reinterpret_cast<PyTypeObject*>(mutable_mapping);
+
+ if (PyType_Ready(&MessageMapContainer_Type) < 0) {
+ return false;
+ }
+
+ PyModule_AddObject(m, "MessageMapContainer",
+ reinterpret_cast<PyObject*>(&MessageMapContainer_Type));
+
+ if (PyType_Ready(&MessageMapIterator_Type) < 0) {
+ return false;
+ }
+
+ PyModule_AddObject(m, "MessageMapIterator",
+ reinterpret_cast<PyObject*>(&MessageMapIterator_Type));
+
ExtensionDict_Type.tp_hash = PyObject_HashNotImplemented;
if (PyType_Ready(&ExtensionDict_Type) < 0) {
return false;
@@ -2478,6 +2712,12 @@ bool InitProto2MessageModule(PyObject *m) {
m, "ExtensionDict",
reinterpret_cast<PyObject*>(&ExtensionDict_Type));
+ // Expose the DescriptorPool used to hold all descriptors added from generated
+ // pb2.py files.
+ Py_INCREF(GetDescriptorPool()); // PyModule_AddObject steals a reference.
+ PyModule_AddObject(
+ m, "default_pool", reinterpret_cast<PyObject*>(GetDescriptorPool()));
+
// This implementation provides full Descriptor types, we advertise it so that
// descriptor.py can use them in replacement of the Python classes.
PyModule_AddIntConstant(m, "_USE_C_DESCRIPTORS", 1);
diff --git a/python/google/protobuf/pyext/message.h b/python/google/protobuf/pyext/message.h
index 2f2da795..7360b207 100644
--- a/python/google/protobuf/pyext/message.h
+++ b/python/google/protobuf/pyext/message.h
@@ -120,7 +120,7 @@ CMessage* NewEmptyMessage(PyObject* type, const Descriptor* descriptor);
// A new message will be created if this is a read-only default instance.
//
// Corresponds to reflection api method ReleaseMessage.
-int ReleaseSubMessage(Message* message,
+int ReleaseSubMessage(CMessage* self,
const FieldDescriptor* field_descriptor,
CMessage* child_cmessage);
@@ -144,7 +144,7 @@ PyObject* InternalGetSubMessage(
// by slice will be removed from cmessage_list by this function.
//
// Corresponds to reflection api method RemoveLast.
-int InternalDeleteRepeatedField(Message* message,
+int InternalDeleteRepeatedField(CMessage* self,
const FieldDescriptor* field_descriptor,
PyObject* slice, PyObject* cmessage_list);
@@ -153,10 +153,15 @@ int InternalSetScalar(CMessage* self,
const FieldDescriptor* field_descriptor,
PyObject* value);
+// Sets the specified scalar value to the message. Requires it is not a Oneof.
+int InternalSetNonOneofScalar(Message* message,
+ const FieldDescriptor* field_descriptor,
+ PyObject* arg);
+
// Retrieves the specified scalar value from the message.
//
// Returns a new python reference.
-PyObject* InternalGetScalar(CMessage* self,
+PyObject* InternalGetScalar(const Message* message,
const FieldDescriptor* field_descriptor);
// Clears the message, removing all contained data. Extension dictionary and
@@ -279,7 +284,7 @@ extern PyObject* kint64min_py;
extern PyObject* kint64max_py;
extern PyObject* kuint64max_py;
-#define C(str) const_cast<char*>(str)
+#define FULL_MODULE_NAME "google.protobuf.pyext._message"
void FormatTypeError(PyObject* arg, char* expected_types);
template<class T>
diff --git a/python/google/protobuf/pyext/message_map_container.cc b/python/google/protobuf/pyext/message_map_container.cc
new file mode 100644
index 00000000..ab8d8fb9
--- /dev/null
+++ b/python/google/protobuf/pyext/message_map_container.cc
@@ -0,0 +1,540 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc. All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: haberman@google.com (Josh Haberman)
+
+#include <google/protobuf/pyext/message_map_container.h>
+
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/message.h>
+#include <google/protobuf/pyext/message.h>
+#include <google/protobuf/pyext/scoped_pyobject_ptr.h>
+
+namespace google {
+namespace protobuf {
+namespace python {
+
+struct MessageMapIterator {
+ PyObject_HEAD;
+
+ // This dict contains the full contents of what we want to iterate over.
+ // There's no way to avoid building this, because the list representation
+ // (which is canonical) can contain duplicate keys. So at the very least we
+ // need a set that lets us skip duplicate keys. And at the point that we're
+ // doing that, we might as well just build the actual dict we're iterating
+ // over and use dict's built-in iterator.
+ PyObject* dict;
+
+ // An iterator on dict.
+ PyObject* iter;
+
+ // A pointer back to the container, so we can notice changes to the version.
+ MessageMapContainer* container;
+
+ // The version of the map when we took the iterator to it.
+ //
+ // We store this so that if the map is modified during iteration we can throw
+ // an error.
+ uint64 version;
+};
+
+static MessageMapIterator* GetIter(PyObject* obj) {
+ return reinterpret_cast<MessageMapIterator*>(obj);
+}
+
+namespace message_map_container {
+
+static MessageMapContainer* GetMap(PyObject* obj) {
+ return reinterpret_cast<MessageMapContainer*>(obj);
+}
+
+// The private constructor of MessageMapContainer objects.
+PyObject* NewContainer(CMessage* parent,
+ const google::protobuf::FieldDescriptor* parent_field_descriptor,
+ PyObject* concrete_class) {
+ if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) {
+ return NULL;
+ }
+
+ PyObject* obj = PyType_GenericAlloc(&MessageMapContainer_Type, 0);
+ if (obj == NULL) {
+ return PyErr_Format(PyExc_RuntimeError,
+ "Could not allocate new container.");
+ }
+
+ MessageMapContainer* self = GetMap(obj);
+
+ self->message = parent->message;
+ self->parent = parent;
+ self->parent_field_descriptor = parent_field_descriptor;
+ self->owner = parent->owner;
+ self->version = 0;
+
+ self->key_field_descriptor =
+ parent_field_descriptor->message_type()->FindFieldByName("key");
+ self->value_field_descriptor =
+ parent_field_descriptor->message_type()->FindFieldByName("value");
+
+ self->message_dict = PyDict_New();
+ if (self->message_dict == NULL) {
+ return PyErr_Format(PyExc_RuntimeError,
+ "Could not allocate message dict.");
+ }
+
+ Py_INCREF(concrete_class);
+ self->subclass_init = concrete_class;
+
+ if (self->key_field_descriptor == NULL ||
+ self->value_field_descriptor == NULL) {
+ Py_DECREF(obj);
+ return PyErr_Format(PyExc_KeyError,
+ "Map entry descriptor did not have key/value fields");
+ }
+
+ return obj;
+}
+
+// Initializes the underlying Message object of "to" so it becomes a new parent
+// repeated scalar, and copies all the values from "from" to it. A child scalar
+// container can be released by passing it as both from and to (e.g. making it
+// the recipient of the new parent message and copying the values from itself).
+static int InitializeAndCopyToParentContainer(
+ MessageMapContainer* from,
+ MessageMapContainer* to) {
+ // For now we require from == to, re-evaluate if we want to support deep copy
+ // as in repeated_composite_container.cc.
+ GOOGLE_DCHECK(from == to);
+ Message* old_message = from->message;
+ Message* new_message = old_message->New();
+ to->parent = NULL;
+ to->parent_field_descriptor = from->parent_field_descriptor;
+ to->message = new_message;
+ to->owner.reset(new_message);
+
+ vector<const FieldDescriptor*> fields;
+ fields.push_back(from->parent_field_descriptor);
+ old_message->GetReflection()->SwapFields(old_message, new_message, fields);
+ return 0;
+}
+
+static PyObject* GetCMessage(MessageMapContainer* self, Message* entry) {
+ // Get or create the CMessage object corresponding to this message.
+ Message* message = entry->GetReflection()->MutableMessage(
+ entry, self->value_field_descriptor);
+ ScopedPyObjectPtr key(PyLong_FromVoidPtr(message));
+ PyObject* ret = PyDict_GetItem(self->message_dict, key);
+
+ if (ret == NULL) {
+ CMessage* cmsg = cmessage::NewEmptyMessage(self->subclass_init,
+ message->GetDescriptor());
+ ret = reinterpret_cast<PyObject*>(cmsg);
+
+ if (cmsg == NULL) {
+ return NULL;
+ }
+ cmsg->owner = self->owner;
+ cmsg->message = message;
+ cmsg->parent = self->parent;
+
+ if (PyDict_SetItem(self->message_dict, key, ret) < 0) {
+ Py_DECREF(ret);
+ return NULL;
+ }
+ } else {
+ Py_INCREF(ret);
+ }
+
+ return ret;
+}
+
+int Release(MessageMapContainer* self) {
+ InitializeAndCopyToParentContainer(self, self);
+ return 0;
+}
+
+void SetOwner(MessageMapContainer* self,
+ const shared_ptr<Message>& new_owner) {
+ self->owner = new_owner;
+}
+
+Py_ssize_t Length(PyObject* _self) {
+ MessageMapContainer* self = GetMap(_self);
+ google::protobuf::Message* message = self->message;
+ return message->GetReflection()->FieldSize(*message,
+ self->parent_field_descriptor);
+}
+
+int MapKeyMatches(MessageMapContainer* self, const Message* entry,
+ PyObject* key) {
+ // TODO(haberman): do we need more strict type checking?
+ ScopedPyObjectPtr entry_key(
+ cmessage::InternalGetScalar(entry, self->key_field_descriptor));
+ int ret = PyObject_RichCompareBool(key, entry_key, Py_EQ);
+ return ret;
+}
+
+int SetItem(PyObject *_self, PyObject *key, PyObject *v) {
+ if (v) {
+ PyErr_Format(PyExc_ValueError,
+ "Direct assignment of submessage not allowed");
+ return -1;
+ }
+
+ // Now we know that this is a delete, not a set.
+
+ MessageMapContainer* self = GetMap(_self);
+ cmessage::AssureWritable(self->parent);
+
+ Message* message = self->message;
+ const Reflection* reflection = message->GetReflection();
+ size_t size =
+ reflection->FieldSize(*message, self->parent_field_descriptor);
+
+ // Right now the Reflection API doesn't support map lookup, so we implement it
+ // via linear search. We need to search from the end because the underlying
+ // representation can have duplicates if a user calls MergeFrom(); the last
+ // one needs to win.
+ //
+ // TODO(haberman): add lookup API to Reflection API.
+ bool found = false;
+ for (int i = size - 1; i >= 0; i--) {
+ Message* entry = reflection->MutableRepeatedMessage(
+ message, self->parent_field_descriptor, i);
+ int matches = MapKeyMatches(self, entry, key);
+ if (matches < 0) return -1;
+ if (matches) {
+ found = true;
+ if (i != size - 1) {
+ reflection->SwapElements(message, self->parent_field_descriptor, i,
+ size - 1);
+ }
+ reflection->RemoveLast(message, self->parent_field_descriptor);
+
+ // Can't exit now, the repeated field representation of maps allows
+ // duplicate keys, and we have to be sure to remove all of them.
+ }
+ }
+
+ if (!found) {
+ PyErr_Format(PyExc_KeyError, "Key not present in map");
+ return -1;
+ }
+
+ self->version++;
+
+ return 0;
+}
+
+PyObject* GetIterator(PyObject *_self) {
+ MessageMapContainer* self = GetMap(_self);
+
+ ScopedPyObjectPtr obj(PyType_GenericAlloc(&MessageMapIterator_Type, 0));
+ if (obj == NULL) {
+ return PyErr_Format(PyExc_KeyError, "Could not allocate iterator");
+ }
+
+ MessageMapIterator* iter = GetIter(obj);
+
+ Py_INCREF(self);
+ iter->container = self;
+ iter->version = self->version;
+ iter->dict = PyDict_New();
+ if (iter->dict == NULL) {
+ return PyErr_Format(PyExc_RuntimeError,
+ "Could not allocate dict for iterator.");
+ }
+
+ // Build the entire map into a dict right now. Start from the beginning so
+ // that later entries win in the case of duplicates.
+ Message* message = self->message;
+ const Reflection* reflection = message->GetReflection();
+
+ // Right now the Reflection API doesn't support map lookup, so we implement it
+ // via linear search. We need to search from the end because the underlying
+ // representation can have duplicates if a user calls MergeFrom(); the last
+ // one needs to win.
+ //
+ // TODO(haberman): add lookup API to Reflection API.
+ size_t size =
+ reflection->FieldSize(*message, self->parent_field_descriptor);
+ for (int i = size - 1; i >= 0; i--) {
+ Message* entry = reflection->MutableRepeatedMessage(
+ message, self->parent_field_descriptor, i);
+ ScopedPyObjectPtr key(
+ cmessage::InternalGetScalar(entry, self->key_field_descriptor));
+ if (PyDict_SetItem(iter->dict, key.get(), GetCMessage(self, entry)) < 0) {
+ return PyErr_Format(PyExc_RuntimeError,
+ "SetItem failed in iterator construction.");
+ }
+ }
+
+ iter->iter = PyObject_GetIter(iter->dict);
+
+ return obj.release();
+}
+
+PyObject* GetItem(PyObject* _self, PyObject* key) {
+ MessageMapContainer* self = GetMap(_self);
+ cmessage::AssureWritable(self->parent);
+ Message* message = self->message;
+ const Reflection* reflection = message->GetReflection();
+
+ // Right now the Reflection API doesn't support map lookup, so we implement it
+ // via linear search. We need to search from the end because the underlying
+ // representation can have duplicates if a user calls MergeFrom(); the last
+ // one needs to win.
+ //
+ // TODO(haberman): add lookup API to Reflection API.
+ size_t size =
+ reflection->FieldSize(*message, self->parent_field_descriptor);
+ for (int i = size - 1; i >= 0; i--) {
+ Message* entry = reflection->MutableRepeatedMessage(
+ message, self->parent_field_descriptor, i);
+ int matches = MapKeyMatches(self, entry, key);
+ if (matches < 0) return NULL;
+ if (matches) {
+ return GetCMessage(self, entry);
+ }
+ }
+
+ // Key is not already present; insert a new entry.
+ Message* entry =
+ reflection->AddMessage(message, self->parent_field_descriptor);
+
+ self->version++;
+
+ if (cmessage::InternalSetNonOneofScalar(entry, self->key_field_descriptor,
+ key) < 0) {
+ reflection->RemoveLast(message, self->parent_field_descriptor);
+ return NULL;
+ }
+
+ return GetCMessage(self, entry);
+}
+
+PyObject* Contains(PyObject* _self, PyObject* key) {
+ MessageMapContainer* self = GetMap(_self);
+ Message* message = self->message;
+ const Reflection* reflection = message->GetReflection();
+
+ // Right now the Reflection API doesn't support map lookup, so we implement it
+ // via linear search.
+ //
+ // TODO(haberman): add lookup API to Reflection API.
+ size_t size =
+ reflection->FieldSize(*message, self->parent_field_descriptor);
+ for (int i = 0; i < size; i++) {
+ Message* entry = reflection->MutableRepeatedMessage(
+ message, self->parent_field_descriptor, i);
+ int matches = MapKeyMatches(self, entry, key);
+ if (matches < 0) return NULL;
+ if (matches) {
+ Py_RETURN_TRUE;
+ }
+ }
+
+ Py_RETURN_FALSE;
+}
+
+PyObject* Clear(PyObject* _self) {
+ MessageMapContainer* self = GetMap(_self);
+ cmessage::AssureWritable(self->parent);
+ Message* message = self->message;
+ const Reflection* reflection = message->GetReflection();
+
+ self->version++;
+ reflection->ClearField(message, self->parent_field_descriptor);
+
+ Py_RETURN_NONE;
+}
+
+PyObject* Get(PyObject* self, PyObject* args) {
+ PyObject* key;
+ PyObject* default_value = NULL;
+ if (PyArg_ParseTuple(args, "O|O", &key, &default_value) < 0) {
+ return NULL;
+ }
+
+ ScopedPyObjectPtr is_present(Contains(self, key));
+ if (is_present.get() == NULL) {
+ return NULL;
+ }
+
+ if (PyObject_IsTrue(is_present.get())) {
+ return GetItem(self, key);
+ } else {
+ if (default_value != NULL) {
+ Py_INCREF(default_value);
+ return default_value;
+ } else {
+ Py_RETURN_NONE;
+ }
+ }
+}
+
+static PyMappingMethods MpMethods = {
+ Length, // mp_length
+ GetItem, // mp_subscript
+ SetItem, // mp_ass_subscript
+};
+
+static void Dealloc(PyObject* _self) {
+ MessageMapContainer* self = GetMap(_self);
+ self->owner.reset();
+ Py_DECREF(self->message_dict);
+ Py_TYPE(_self)->tp_free(_self);
+}
+
+static PyMethodDef Methods[] = {
+ { "__contains__", (PyCFunction)Contains, METH_O,
+ "Tests whether the map contains this element."},
+ { "clear", (PyCFunction)Clear, METH_NOARGS,
+ "Removes all elements from the map."},
+ { "get", Get, METH_VARARGS,
+ "Gets the value for the given key if present, or otherwise a default" },
+ { "get_or_create", GetItem, METH_O,
+ "Alias for getitem, useful to make explicit that the map is mutated." },
+ /*
+ { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS,
+ "Makes a deep copy of the class." },
+ { "__reduce__", (PyCFunction)Reduce, METH_NOARGS,
+ "Outputs picklable representation of the repeated field." },
+ */
+ {NULL, NULL},
+};
+
+} // namespace message_map_container
+
+namespace message_map_iterator {
+
+static void Dealloc(PyObject* _self) {
+ MessageMapIterator* self = GetIter(_self);
+ Py_DECREF(self->dict);
+ Py_DECREF(self->iter);
+ Py_DECREF(self->container);
+ Py_TYPE(_self)->tp_free(_self);
+}
+
+PyObject* IterNext(PyObject* _self) {
+ MessageMapIterator* self = GetIter(_self);
+
+ // This won't catch mutations to the map performed by MergeFrom(); no easy way
+ // to address that.
+ if (self->version != self->container->version) {
+ return PyErr_Format(PyExc_RuntimeError,
+ "Map modified during iteration.");
+ }
+
+ return PyIter_Next(self->iter);
+}
+
+} // namespace message_map_iterator
+
+PyTypeObject MessageMapContainer_Type = {
+ PyVarObject_HEAD_INIT(&PyType_Type, 0)
+ FULL_MODULE_NAME ".MessageMapContainer", // tp_name
+ sizeof(MessageMapContainer), // tp_basicsize
+ 0, // tp_itemsize
+ message_map_container::Dealloc, // tp_dealloc
+ 0, // tp_print
+ 0, // tp_getattr
+ 0, // tp_setattr
+ 0, // tp_compare
+ 0, // tp_repr
+ 0, // tp_as_number
+ 0, // tp_as_sequence
+ &message_map_container::MpMethods, // tp_as_mapping
+ 0, // tp_hash
+ 0, // tp_call
+ 0, // tp_str
+ 0, // tp_getattro
+ 0, // tp_setattro
+ 0, // tp_as_buffer
+ Py_TPFLAGS_DEFAULT, // tp_flags
+ "A map container for message", // tp_doc
+ 0, // tp_traverse
+ 0, // tp_clear
+ 0, // tp_richcompare
+ 0, // tp_weaklistoffset
+ message_map_container::GetIterator, // tp_iter
+ 0, // tp_iternext
+ message_map_container::Methods, // tp_methods
+ 0, // tp_members
+ 0, // tp_getset
+ 0, // tp_base
+ 0, // tp_dict
+ 0, // tp_descr_get
+ 0, // tp_descr_set
+ 0, // tp_dictoffset
+ 0, // tp_init
+};
+
+PyTypeObject MessageMapIterator_Type = {
+ PyVarObject_HEAD_INIT(&PyType_Type, 0)
+ FULL_MODULE_NAME ".MessageMapIterator", // tp_name
+ sizeof(MessageMapIterator), // tp_basicsize
+ 0, // tp_itemsize
+ message_map_iterator::Dealloc, // tp_dealloc
+ 0, // tp_print
+ 0, // tp_getattr
+ 0, // tp_setattr
+ 0, // tp_compare
+ 0, // tp_repr
+ 0, // tp_as_number
+ 0, // tp_as_sequence
+ 0, // tp_as_mapping
+ 0, // tp_hash
+ 0, // tp_call
+ 0, // tp_str
+ 0, // tp_getattro
+ 0, // tp_setattro
+ 0, // tp_as_buffer
+ Py_TPFLAGS_DEFAULT, // tp_flags
+ "A scalar map iterator", // tp_doc
+ 0, // tp_traverse
+ 0, // tp_clear
+ 0, // tp_richcompare
+ 0, // tp_weaklistoffset
+ PyObject_SelfIter, // tp_iter
+ message_map_iterator::IterNext, // tp_iternext
+ 0, // tp_methods
+ 0, // tp_members
+ 0, // tp_getset
+ 0, // tp_base
+ 0, // tp_dict
+ 0, // tp_descr_get
+ 0, // tp_descr_set
+ 0, // tp_dictoffset
+ 0, // tp_init
+};
+
+} // namespace python
+} // namespace protobuf
+} // namespace google
diff --git a/python/google/protobuf/pyext/message_map_container.h b/python/google/protobuf/pyext/message_map_container.h
new file mode 100644
index 00000000..4ca0aecc
--- /dev/null
+++ b/python/google/protobuf/pyext/message_map_container.h
@@ -0,0 +1,117 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc. All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_MAP_CONTAINER_H__
+#define GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_MAP_CONTAINER_H__
+
+#include <Python.h>
+
+#include <memory>
+#ifndef _SHARED_PTR_H
+#include <google/protobuf/stubs/shared_ptr.h>
+#endif
+
+#include <google/protobuf/descriptor.h>
+
+namespace google {
+namespace protobuf {
+
+class Message;
+
+using internal::shared_ptr;
+
+namespace python {
+
+struct CMessage;
+
+struct MessageMapContainer {
+ PyObject_HEAD;
+
+ // This is the top-level C++ Message object that owns the whole
+ // proto tree. Every Python MessageMapContainer holds a
+ // reference to it in order to keep it alive as long as there's a
+ // Python object that references any part of the tree.
+ shared_ptr<Message> owner;
+
+ // Pointer to the C++ Message that contains this container. The
+ // MessageMapContainer does not own this pointer.
+ Message* message;
+
+ // Weak reference to a parent CMessage object (i.e. may be NULL.)
+ //
+ // Used to make sure all ancestors are also mutable when first
+ // modifying the container.
+ CMessage* parent;
+
+ // Pointer to the parent's descriptor that describes this
+ // field. Used together with the parent's message when making a
+ // default message instance mutable.
+ // The pointer is owned by the global DescriptorPool.
+ const FieldDescriptor* parent_field_descriptor;
+ const FieldDescriptor* key_field_descriptor;
+ const FieldDescriptor* value_field_descriptor;
+
+ // A callable that is used to create new child messages.
+ PyObject* subclass_init;
+
+ // A dict mapping Message* -> CMessage.
+ PyObject* message_dict;
+
+ // We bump this whenever we perform a mutation, to invalidate existing
+ // iterators.
+ uint64 version;
+};
+
+extern PyTypeObject MessageMapContainer_Type;
+extern PyTypeObject MessageMapIterator_Type;
+
+namespace message_map_container {
+
+// Builds a MessageMapContainer object, from a parent message and a
+// field descriptor.
+extern PyObject* NewContainer(CMessage* parent,
+ const FieldDescriptor* parent_field_descriptor,
+ PyObject* concrete_class);
+
+// Releases the messages in the container to a new message.
+//
+// Returns 0 on success, -1 on failure.
+int Release(MessageMapContainer* self);
+
+// Set the owner field of self and any children of self.
+void SetOwner(MessageMapContainer* self,
+ const shared_ptr<Message>& new_owner);
+
+} // namespace message_map_container
+} // namespace python
+} // namespace protobuf
+
+} // namespace google
+#endif // GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_MAP_CONTAINER_H__
diff --git a/python/google/protobuf/pyext/repeated_composite_container.cc b/python/google/protobuf/pyext/repeated_composite_container.cc
index 0fe98e73..86b75d0f 100644
--- a/python/google/protobuf/pyext/repeated_composite_container.cc
+++ b/python/google/protobuf/pyext/repeated_composite_container.cc
@@ -367,8 +367,8 @@ int AssignSubscript(RepeatedCompositeContainer* self,
}
// Delete from the underlying Message, if any.
- if (self->message != NULL) {
- if (cmessage::InternalDeleteRepeatedField(self->message,
+ if (self->parent != NULL) {
+ if (cmessage::InternalDeleteRepeatedField(self->parent,
self->parent_field_descriptor,
slice,
self->child_messages) < 0) {
@@ -572,47 +572,35 @@ static PyObject* Pop(RepeatedCompositeContainer* self,
return item;
}
-// The caller takes ownership of the returned Message.
-Message* ReleaseLast(const FieldDescriptor* field,
- const Descriptor* type,
- Message* message) {
+// Release field of parent message and transfer the ownership to target.
+void ReleaseLastTo(CMessage* parent,
+ const FieldDescriptor* field,
+ CMessage* target) {
+ GOOGLE_CHECK_NOTNULL(parent);
GOOGLE_CHECK_NOTNULL(field);
- GOOGLE_CHECK_NOTNULL(type);
- GOOGLE_CHECK_NOTNULL(message);
+ GOOGLE_CHECK_NOTNULL(target);
- Message* released_message = message->GetReflection()->ReleaseLast(
- message, field);
+ shared_ptr<Message> released_message(
+ parent->message->GetReflection()->ReleaseLast(parent->message, field));
// TODO(tibell): Deal with proto1.
// ReleaseMessage will return NULL which differs from
// child_cmessage->message, if the field does not exist. In this case,
// the latter points to the default instance via a const_cast<>, so we
// have to reset it to a new mutable object since we are taking ownership.
- if (released_message == NULL) {
+ if (released_message.get() == NULL) {
const Message* prototype =
- cmessage::GetMessageFactory()->GetPrototype(type);
+ cmessage::GetMessageFactory()->GetPrototype(
+ target->message->GetDescriptor());
GOOGLE_CHECK_NOTNULL(prototype);
- return prototype->New();
- } else {
- return released_message;
+ released_message.reset(prototype->New());
}
-}
-// Release field of message and transfer the ownership to cmessage.
-void ReleaseLastTo(const FieldDescriptor* field,
- Message* message,
- CMessage* cmessage) {
- GOOGLE_CHECK_NOTNULL(field);
- GOOGLE_CHECK_NOTNULL(message);
- GOOGLE_CHECK_NOTNULL(cmessage);
-
- shared_ptr<Message> released_message(
- ReleaseLast(field, cmessage->message->GetDescriptor(), message));
- cmessage->parent = NULL;
- cmessage->parent_field_descriptor = NULL;
- cmessage->message = released_message.get();
- cmessage->read_only = false;
- cmessage::SetOwner(cmessage, released_message);
+ target->parent = NULL;
+ target->parent_field_descriptor = NULL;
+ target->message = released_message.get();
+ target->read_only = false;
+ cmessage::SetOwner(target, released_message);
}
// Called to release a container using
@@ -635,7 +623,7 @@ int Release(RepeatedCompositeContainer* self) {
for (Py_ssize_t i = size - 1; i >= 0; --i) {
CMessage* child_cmessage = reinterpret_cast<CMessage*>(
PyList_GET_ITEM(self->child_messages, i));
- ReleaseLastTo(field, message, child_cmessage);
+ ReleaseLastTo(self->parent, field, child_cmessage);
}
// Detach from containing message.
@@ -732,9 +720,7 @@ static PyMethodDef Methods[] = {
PyTypeObject RepeatedCompositeContainer_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
- // Keep the fully qualified _message symbol in a line for opensource.
- "google.protobuf.pyext._message."
- "RepeatedCompositeContainer", // tp_name
+ FULL_MODULE_NAME ".RepeatedCompositeContainer", // tp_name
sizeof(RepeatedCompositeContainer), // tp_basicsize
0, // tp_itemsize
(destructor)repeated_composite_container::Dealloc, // tp_dealloc
diff --git a/python/google/protobuf/pyext/repeated_composite_container.h b/python/google/protobuf/pyext/repeated_composite_container.h
index ce7cee0f..e0f21360 100644
--- a/python/google/protobuf/pyext/repeated_composite_container.h
+++ b/python/google/protobuf/pyext/repeated_composite_container.h
@@ -161,13 +161,13 @@ int SetOwner(RepeatedCompositeContainer* self,
const shared_ptr<Message>& new_owner);
// Removes the last element of the repeated message field 'field' on
-// the Message 'message', and transfers the ownership of the released
-// Message to 'cmessage'.
+// the Message 'parent', and transfers the ownership of the released
+// Message to 'target'.
//
// Corresponds to reflection api method ReleaseMessage.
-void ReleaseLastTo(const FieldDescriptor* field,
- Message* message,
- CMessage* cmessage);
+void ReleaseLastTo(CMessage* parent,
+ const FieldDescriptor* field,
+ CMessage* target);
} // namespace repeated_composite_container
} // namespace python
diff --git a/python/google/protobuf/pyext/repeated_scalar_container.cc b/python/google/protobuf/pyext/repeated_scalar_container.cc
index 110a4c85..fd196836 100644
--- a/python/google/protobuf/pyext/repeated_scalar_container.cc
+++ b/python/google/protobuf/pyext/repeated_scalar_container.cc
@@ -102,7 +102,7 @@ static int AssignItem(RepeatedScalarContainer* self,
if (arg == NULL) {
ScopedPyObjectPtr py_index(PyLong_FromLong(index));
- return cmessage::InternalDeleteRepeatedField(message, field_descriptor,
+ return cmessage::InternalDeleteRepeatedField(self->parent, field_descriptor,
py_index, NULL);
}
@@ -470,7 +470,7 @@ static int AssSubscript(RepeatedScalarContainer* self,
if (value == NULL) {
return cmessage::InternalDeleteRepeatedField(
- message, field_descriptor, slice, NULL);
+ self->parent, field_descriptor, slice, NULL);
}
if (!create_list) {
@@ -769,9 +769,7 @@ static PyMethodDef Methods[] = {
PyTypeObject RepeatedScalarContainer_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
- // Keep the fully qualified _message symbol in a line for opensource.
- "google.protobuf.pyext._message."
- "RepeatedScalarContainer", // tp_name
+ FULL_MODULE_NAME ".RepeatedScalarContainer", // tp_name
sizeof(RepeatedScalarContainer), // tp_basicsize
0, // tp_itemsize
(destructor)repeated_scalar_container::Dealloc, // tp_dealloc
diff --git a/python/google/protobuf/pyext/scalar_map_container.cc b/python/google/protobuf/pyext/scalar_map_container.cc
new file mode 100644
index 00000000..6f731d27
--- /dev/null
+++ b/python/google/protobuf/pyext/scalar_map_container.cc
@@ -0,0 +1,514 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc. All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: haberman@google.com (Josh Haberman)
+
+#include <google/protobuf/pyext/scalar_map_container.h>
+
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/message.h>
+#include <google/protobuf/pyext/message.h>
+#include <google/protobuf/pyext/scoped_pyobject_ptr.h>
+
+namespace google {
+namespace protobuf {
+namespace python {
+
+struct ScalarMapIterator {
+ PyObject_HEAD;
+
+ // This dict contains the full contents of what we want to iterate over.
+ // There's no way to avoid building this, because the list representation
+ // (which is canonical) can contain duplicate keys. So at the very least we
+ // need a set that lets us skip duplicate keys. And at the point that we're
+ // doing that, we might as well just build the actual dict we're iterating
+ // over and use dict's built-in iterator.
+ PyObject* dict;
+
+ // An iterator on dict.
+ PyObject* iter;
+
+ // A pointer back to the container, so we can notice changes to the version.
+ ScalarMapContainer* container;
+
+ // The version of the map when we took the iterator to it.
+ //
+ // We store this so that if the map is modified during iteration we can throw
+ // an error.
+ uint64 version;
+};
+
+static ScalarMapIterator* GetIter(PyObject* obj) {
+ return reinterpret_cast<ScalarMapIterator*>(obj);
+}
+
+namespace scalar_map_container {
+
+static ScalarMapContainer* GetMap(PyObject* obj) {
+ return reinterpret_cast<ScalarMapContainer*>(obj);
+}
+
+// The private constructor of ScalarMapContainer objects.
+PyObject *NewContainer(
+ CMessage* parent, const google::protobuf::FieldDescriptor* parent_field_descriptor) {
+ if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) {
+ return NULL;
+ }
+
+ ScopedPyObjectPtr obj(PyType_GenericAlloc(&ScalarMapContainer_Type, 0));
+ if (obj.get() == NULL) {
+ return PyErr_Format(PyExc_RuntimeError,
+ "Could not allocate new container.");
+ }
+
+ ScalarMapContainer* self = GetMap(obj);
+
+ self->message = parent->message;
+ self->parent = parent;
+ self->parent_field_descriptor = parent_field_descriptor;
+ self->owner = parent->owner;
+ self->version = 0;
+
+ self->key_field_descriptor =
+ parent_field_descriptor->message_type()->FindFieldByName("key");
+ self->value_field_descriptor =
+ parent_field_descriptor->message_type()->FindFieldByName("value");
+
+ if (self->key_field_descriptor == NULL ||
+ self->value_field_descriptor == NULL) {
+ return PyErr_Format(PyExc_KeyError,
+ "Map entry descriptor did not have key/value fields");
+ }
+
+ return obj.release();
+}
+
+// Initializes the underlying Message object of "to" so it becomes a new parent
+// repeated scalar, and copies all the values from "from" to it. A child scalar
+// container can be released by passing it as both from and to (e.g. making it
+// the recipient of the new parent message and copying the values from itself).
+static int InitializeAndCopyToParentContainer(
+ ScalarMapContainer* from,
+ ScalarMapContainer* to) {
+ // For now we require from == to, re-evaluate if we want to support deep copy
+ // as in repeated_scalar_container.cc.
+ GOOGLE_DCHECK(from == to);
+ Message* old_message = from->message;
+ Message* new_message = old_message->New();
+ to->parent = NULL;
+ to->parent_field_descriptor = from->parent_field_descriptor;
+ to->message = new_message;
+ to->owner.reset(new_message);
+
+ vector<const FieldDescriptor*> fields;
+ fields.push_back(from->parent_field_descriptor);
+ old_message->GetReflection()->SwapFields(old_message, new_message, fields);
+ return 0;
+}
+
+int Release(ScalarMapContainer* self) {
+ return InitializeAndCopyToParentContainer(self, self);
+}
+
+void SetOwner(ScalarMapContainer* self,
+ const shared_ptr<Message>& new_owner) {
+ self->owner = new_owner;
+}
+
+Py_ssize_t Length(PyObject* _self) {
+ ScalarMapContainer* self = GetMap(_self);
+ google::protobuf::Message* message = self->message;
+ return message->GetReflection()->FieldSize(*message,
+ self->parent_field_descriptor);
+}
+
+int MapKeyMatches(ScalarMapContainer* self, const Message* entry,
+ PyObject* key) {
+ // TODO(haberman): do we need more strict type checking?
+ ScopedPyObjectPtr entry_key(
+ cmessage::InternalGetScalar(entry, self->key_field_descriptor));
+ int ret = PyObject_RichCompareBool(key, entry_key, Py_EQ);
+ return ret;
+}
+
+PyObject* GetItem(PyObject* _self, PyObject* key) {
+ ScalarMapContainer* self = GetMap(_self);
+
+ Message* message = self->message;
+ const Reflection* reflection = message->GetReflection();
+
+ // Right now the Reflection API doesn't support map lookup, so we implement it
+ // via linear search.
+ //
+ // TODO(haberman): add lookup API to Reflection API.
+ size_t size = reflection->FieldSize(*message, self->parent_field_descriptor);
+ for (int i = size - 1; i >= 0; i--) {
+ const Message& entry = reflection->GetRepeatedMessage(
+ *message, self->parent_field_descriptor, i);
+ int matches = MapKeyMatches(self, &entry, key);
+ if (matches < 0) return NULL;
+ if (matches) {
+ return cmessage::InternalGetScalar(&entry, self->value_field_descriptor);
+ }
+ }
+
+ // Need to add a new entry.
+ Message* entry =
+ reflection->AddMessage(message, self->parent_field_descriptor);
+ PyObject* ret = NULL;
+
+ if (cmessage::InternalSetNonOneofScalar(entry, self->key_field_descriptor,
+ key) >= 0) {
+ ret = cmessage::InternalGetScalar(entry, self->value_field_descriptor);
+ }
+
+ self->version++;
+
+ // If there was a type error above, it set the Python exception.
+ return ret;
+}
+
+int SetItem(PyObject *_self, PyObject *key, PyObject *v) {
+ ScalarMapContainer* self = GetMap(_self);
+ cmessage::AssureWritable(self->parent);
+
+ Message* message = self->message;
+ const Reflection* reflection = message->GetReflection();
+ size_t size =
+ reflection->FieldSize(*message, self->parent_field_descriptor);
+ self->version++;
+
+ if (v) {
+ // Set item.
+ //
+ // Right now the Reflection API doesn't support map lookup, so we implement
+ // it via linear search.
+ //
+ // TODO(haberman): add lookup API to Reflection API.
+ for (int i = size - 1; i >= 0; i--) {
+ Message* entry = reflection->MutableRepeatedMessage(
+ message, self->parent_field_descriptor, i);
+ int matches = MapKeyMatches(self, entry, key);
+ if (matches < 0) return -1;
+ if (matches) {
+ return cmessage::InternalSetNonOneofScalar(
+ entry, self->value_field_descriptor, v);
+ }
+ }
+
+ // Key is not already present; insert a new entry.
+ Message* entry =
+ reflection->AddMessage(message, self->parent_field_descriptor);
+
+ if (cmessage::InternalSetNonOneofScalar(entry, self->key_field_descriptor,
+ key) < 0 ||
+ cmessage::InternalSetNonOneofScalar(entry, self->value_field_descriptor,
+ v) < 0) {
+ reflection->RemoveLast(message, self->parent_field_descriptor);
+ return -1;
+ }
+
+ return 0;
+ } else {
+ bool found = false;
+ for (int i = size - 1; i >= 0; i--) {
+ Message* entry = reflection->MutableRepeatedMessage(
+ message, self->parent_field_descriptor, i);
+ int matches = MapKeyMatches(self, entry, key);
+ if (matches < 0) return -1;
+ if (matches) {
+ found = true;
+ if (i != size - 1) {
+ reflection->SwapElements(message, self->parent_field_descriptor, i,
+ size - 1);
+ }
+ reflection->RemoveLast(message, self->parent_field_descriptor);
+
+ // Can't exit now, the repeated field representation of maps allows
+ // duplicate keys, and we have to be sure to remove all of them.
+ }
+ }
+
+ if (found) {
+ return 0;
+ } else {
+ PyErr_Format(PyExc_KeyError, "Key not present in map");
+ return -1;
+ }
+ }
+}
+
+PyObject* GetIterator(PyObject *_self) {
+ ScalarMapContainer* self = GetMap(_self);
+
+ ScopedPyObjectPtr obj(PyType_GenericAlloc(&ScalarMapIterator_Type, 0));
+ if (obj == NULL) {
+ return PyErr_Format(PyExc_KeyError, "Could not allocate iterator");
+ }
+
+ ScalarMapIterator* iter = GetIter(obj.get());
+
+ Py_INCREF(self);
+ iter->container = self;
+ iter->version = self->version;
+ iter->dict = PyDict_New();
+ if (iter->dict == NULL) {
+ return PyErr_Format(PyExc_RuntimeError,
+ "Could not allocate dict for iterator.");
+ }
+
+ // Build the entire map into a dict right now. Start from the beginning so
+ // that later entries win in the case of duplicates.
+ Message* message = self->message;
+ const Reflection* reflection = message->GetReflection();
+
+ // Right now the Reflection API doesn't support map lookup, so we implement it
+ // via linear search. We need to search from the end because the underlying
+ // representation can have duplicates if a user calls MergeFrom(); the last
+ // one needs to win.
+ //
+ // TODO(haberman): add lookup API to Reflection API.
+ size_t size =
+ reflection->FieldSize(*message, self->parent_field_descriptor);
+ for (int i = 0; i < size; i++) {
+ Message* entry = reflection->MutableRepeatedMessage(
+ message, self->parent_field_descriptor, i);
+ ScopedPyObjectPtr key(
+ cmessage::InternalGetScalar(entry, self->key_field_descriptor));
+ ScopedPyObjectPtr val(
+ cmessage::InternalGetScalar(entry, self->value_field_descriptor));
+ if (PyDict_SetItem(iter->dict, key.get(), val.get()) < 0) {
+ return PyErr_Format(PyExc_RuntimeError,
+ "SetItem failed in iterator construction.");
+ }
+ }
+
+
+ iter->iter = PyObject_GetIter(iter->dict);
+
+
+ return obj.release();
+}
+
+PyObject* Clear(PyObject* _self) {
+ ScalarMapContainer* self = GetMap(_self);
+ cmessage::AssureWritable(self->parent);
+ Message* message = self->message;
+ const Reflection* reflection = message->GetReflection();
+
+ reflection->ClearField(message, self->parent_field_descriptor);
+
+ Py_RETURN_NONE;
+}
+
+PyObject* Contains(PyObject* _self, PyObject* key) {
+ ScalarMapContainer* self = GetMap(_self);
+
+ Message* message = self->message;
+ const Reflection* reflection = message->GetReflection();
+
+ // Right now the Reflection API doesn't support map lookup, so we implement it
+ // via linear search.
+ //
+ // TODO(haberman): add lookup API to Reflection API.
+ size_t size = reflection->FieldSize(*message, self->parent_field_descriptor);
+ for (int i = size - 1; i >= 0; i--) {
+ const Message& entry = reflection->GetRepeatedMessage(
+ *message, self->parent_field_descriptor, i);
+ int matches = MapKeyMatches(self, &entry, key);
+ if (matches < 0) return NULL;
+ if (matches) {
+ Py_RETURN_TRUE;
+ }
+ }
+
+ Py_RETURN_FALSE;
+}
+
+PyObject* Get(PyObject* self, PyObject* args) {
+ PyObject* key;
+ PyObject* default_value = NULL;
+ if (PyArg_ParseTuple(args, "O|O", &key, &default_value) < 0) {
+ return NULL;
+ }
+
+ ScopedPyObjectPtr is_present(Contains(self, key));
+ if (is_present.get() == NULL) {
+ return NULL;
+ }
+
+ if (PyObject_IsTrue(is_present.get())) {
+ return GetItem(self, key);
+ } else {
+ if (default_value != NULL) {
+ Py_INCREF(default_value);
+ return default_value;
+ } else {
+ Py_RETURN_NONE;
+ }
+ }
+}
+
+static PyMappingMethods MpMethods = {
+ Length, // mp_length
+ GetItem, // mp_subscript
+ SetItem, // mp_ass_subscript
+};
+
+static void Dealloc(PyObject* _self) {
+ ScalarMapContainer* self = GetMap(_self);
+ self->owner.reset();
+ Py_TYPE(_self)->tp_free(_self);
+}
+
+static PyMethodDef Methods[] = {
+ { "__contains__", Contains, METH_O,
+ "Tests whether a key is a member of the map." },
+ { "clear", (PyCFunction)Clear, METH_NOARGS,
+ "Removes all elements from the map." },
+ { "get", Get, METH_VARARGS,
+ "Gets the value for the given key if present, or otherwise a default" },
+ /*
+ { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS,
+ "Makes a deep copy of the class." },
+ { "__reduce__", (PyCFunction)Reduce, METH_NOARGS,
+ "Outputs picklable representation of the repeated field." },
+ */
+ {NULL, NULL},
+};
+
+} // namespace scalar_map_container
+
+namespace scalar_map_iterator {
+
+static void Dealloc(PyObject* _self) {
+ ScalarMapIterator* self = GetIter(_self);
+ Py_DECREF(self->dict);
+ Py_DECREF(self->iter);
+ Py_DECREF(self->container);
+ Py_TYPE(_self)->tp_free(_self);
+}
+
+PyObject* IterNext(PyObject* _self) {
+ ScalarMapIterator* self = GetIter(_self);
+
+ // This won't catch mutations to the map performed by MergeFrom(); no easy way
+ // to address that.
+ if (self->version != self->container->version) {
+ return PyErr_Format(PyExc_RuntimeError,
+ "Map modified during iteration.");
+ }
+
+ return PyIter_Next(self->iter);
+}
+
+} // namespace scalar_map_iterator
+
+PyTypeObject ScalarMapContainer_Type = {
+ PyVarObject_HEAD_INIT(&PyType_Type, 0)
+ FULL_MODULE_NAME ".ScalarMapContainer", // tp_name
+ sizeof(ScalarMapContainer), // tp_basicsize
+ 0, // tp_itemsize
+ scalar_map_container::Dealloc, // tp_dealloc
+ 0, // tp_print
+ 0, // tp_getattr
+ 0, // tp_setattr
+ 0, // tp_compare
+ 0, // tp_repr
+ 0, // tp_as_number
+ 0, // tp_as_sequence
+ &scalar_map_container::MpMethods, // tp_as_mapping
+ 0, // tp_hash
+ 0, // tp_call
+ 0, // tp_str
+ 0, // tp_getattro
+ 0, // tp_setattro
+ 0, // tp_as_buffer
+ Py_TPFLAGS_DEFAULT, // tp_flags
+ "A scalar map container", // tp_doc
+ 0, // tp_traverse
+ 0, // tp_clear
+ 0, // tp_richcompare
+ 0, // tp_weaklistoffset
+ scalar_map_container::GetIterator, // tp_iter
+ 0, // tp_iternext
+ scalar_map_container::Methods, // tp_methods
+ 0, // tp_members
+ 0, // tp_getset
+ 0, // tp_base
+ 0, // tp_dict
+ 0, // tp_descr_get
+ 0, // tp_descr_set
+ 0, // tp_dictoffset
+ 0, // tp_init
+};
+
+PyTypeObject ScalarMapIterator_Type = {
+ PyVarObject_HEAD_INIT(&PyType_Type, 0)
+ FULL_MODULE_NAME ".ScalarMapIterator", // tp_name
+ sizeof(ScalarMapIterator), // tp_basicsize
+ 0, // tp_itemsize
+ scalar_map_iterator::Dealloc, // tp_dealloc
+ 0, // tp_print
+ 0, // tp_getattr
+ 0, // tp_setattr
+ 0, // tp_compare
+ 0, // tp_repr
+ 0, // tp_as_number
+ 0, // tp_as_sequence
+ 0, // tp_as_mapping
+ 0, // tp_hash
+ 0, // tp_call
+ 0, // tp_str
+ 0, // tp_getattro
+ 0, // tp_setattro
+ 0, // tp_as_buffer
+ Py_TPFLAGS_DEFAULT, // tp_flags
+ "A scalar map iterator", // tp_doc
+ 0, // tp_traverse
+ 0, // tp_clear
+ 0, // tp_richcompare
+ 0, // tp_weaklistoffset
+ PyObject_SelfIter, // tp_iter
+ scalar_map_iterator::IterNext, // tp_iternext
+ 0, // tp_methods
+ 0, // tp_members
+ 0, // tp_getset
+ 0, // tp_base
+ 0, // tp_dict
+ 0, // tp_descr_get
+ 0, // tp_descr_set
+ 0, // tp_dictoffset
+ 0, // tp_init
+};
+
+} // namespace python
+} // namespace protobuf
+} // namespace google
diff --git a/python/google/protobuf/pyext/scalar_map_container.h b/python/google/protobuf/pyext/scalar_map_container.h
new file mode 100644
index 00000000..254e6e98
--- /dev/null
+++ b/python/google/protobuf/pyext/scalar_map_container.h
@@ -0,0 +1,110 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc. All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_SCALAR_MAP_CONTAINER_H__
+#define GOOGLE_PROTOBUF_PYTHON_CPP_SCALAR_MAP_CONTAINER_H__
+
+#include <Python.h>
+
+#include <memory>
+#ifndef _SHARED_PTR_H
+#include <google/protobuf/stubs/shared_ptr.h>
+#endif
+
+#include <google/protobuf/descriptor.h>
+
+namespace google {
+namespace protobuf {
+
+class Message;
+
+using internal::shared_ptr;
+
+namespace python {
+
+struct CMessage;
+
+struct ScalarMapContainer {
+ PyObject_HEAD;
+
+ // This is the top-level C++ Message object that owns the whole
+ // proto tree. Every Python ScalarMapContainer holds a
+ // reference to it in order to keep it alive as long as there's a
+ // Python object that references any part of the tree.
+ shared_ptr<Message> owner;
+
+ // Pointer to the C++ Message that contains this container. The
+ // ScalarMapContainer does not own this pointer.
+ Message* message;
+
+ // Weak reference to a parent CMessage object (i.e. may be NULL.)
+ //
+ // Used to make sure all ancestors are also mutable when first
+ // modifying the container.
+ CMessage* parent;
+
+ // Pointer to the parent's descriptor that describes this
+ // field. Used together with the parent's message when making a
+ // default message instance mutable.
+ // The pointer is owned by the global DescriptorPool.
+ const FieldDescriptor* parent_field_descriptor;
+ const FieldDescriptor* key_field_descriptor;
+ const FieldDescriptor* value_field_descriptor;
+
+ // We bump this whenever we perform a mutation, to invalidate existing
+ // iterators.
+ uint64 version;
+};
+
+extern PyTypeObject ScalarMapContainer_Type;
+extern PyTypeObject ScalarMapIterator_Type;
+
+namespace scalar_map_container {
+
+// Builds a ScalarMapContainer object, from a parent message and a
+// field descriptor.
+extern PyObject *NewContainer(
+ CMessage* parent, const FieldDescriptor* parent_field_descriptor);
+
+// Releases the messages in the container to a new message.
+//
+// Returns 0 on success, -1 on failure.
+int Release(ScalarMapContainer* self);
+
+// Set the owner field of self and any children of self.
+void SetOwner(ScalarMapContainer* self,
+ const shared_ptr<Message>& new_owner);
+
+} // namespace scalar_map_container
+} // namespace python
+} // namespace protobuf
+
+} // namespace google
+#endif // GOOGLE_PROTOBUF_PYTHON_CPP_SCALAR_MAP_CONTAINER_H__