diff options
author | Bo Yang <teboring@google.com> | 2015-05-21 14:28:59 -0700 |
---|---|---|
committer | Bo Yang <teboring@google.com> | 2015-05-21 19:32:02 -0700 |
commit | 5db217305f37a79eeccd70f000088a06ec82fcec (patch) | |
tree | be53dcf0c0b47ef9178ab8a6fa5c1946ee84a28f /python/google/protobuf/pyext | |
parent | 56095026ccc2f755a6fdb296e30c3ddec8f556a2 (diff) | |
download | protobuf-5db217305f37a79eeccd70f000088a06ec82fcec.tar.gz protobuf-5db217305f37a79eeccd70f000088a06ec82fcec.tar.bz2 protobuf-5db217305f37a79eeccd70f000088a06ec82fcec.zip |
down-integrate internal changes
Diffstat (limited to 'python/google/protobuf/pyext')
-rw-r--r-- | python/google/protobuf/pyext/descriptor.cc | 287 | ||||
-rw-r--r-- | python/google/protobuf/pyext/descriptor.h | 21 | ||||
-rw-r--r-- | python/google/protobuf/pyext/descriptor_containers.cc | 26 | ||||
-rw-r--r-- | python/google/protobuf/pyext/descriptor_containers.h | 5 | ||||
-rw-r--r-- | python/google/protobuf/pyext/descriptor_pool.cc | 185 | ||||
-rw-r--r-- | python/google/protobuf/pyext/descriptor_pool.h | 8 | ||||
-rw-r--r-- | python/google/protobuf/pyext/extension_dict.cc | 17 | ||||
-rw-r--r-- | python/google/protobuf/pyext/message.cc | 432 | ||||
-rw-r--r-- | python/google/protobuf/pyext/message.h | 13 | ||||
-rw-r--r-- | python/google/protobuf/pyext/message_map_container.cc | 540 | ||||
-rw-r--r-- | python/google/protobuf/pyext/message_map_container.h | 117 | ||||
-rw-r--r-- | python/google/protobuf/pyext/repeated_composite_container.cc | 56 | ||||
-rw-r--r-- | python/google/protobuf/pyext/repeated_composite_container.h | 10 | ||||
-rw-r--r-- | python/google/protobuf/pyext/repeated_scalar_container.cc | 8 | ||||
-rw-r--r-- | python/google/protobuf/pyext/scalar_map_container.cc | 514 | ||||
-rw-r--r-- | python/google/protobuf/pyext/scalar_map_container.h | 110 |
16 files changed, 1948 insertions, 401 deletions
diff --git a/python/google/protobuf/pyext/descriptor.cc b/python/google/protobuf/pyext/descriptor.cc index e77d0bb9..2160757b 100644 --- a/python/google/protobuf/pyext/descriptor.cc +++ b/python/google/protobuf/pyext/descriptor.cc @@ -43,8 +43,6 @@ #include <google/protobuf/pyext/message.h> #include <google/protobuf/pyext/scoped_pyobject_ptr.h> -#define C(str) const_cast<char*>(str) - #if PY_MAJOR_VERSION >= 3 #define PyString_FromStringAndSize PyUnicode_FromStringAndSize #define PyString_Check PyUnicode_Check @@ -257,8 +255,14 @@ namespace descriptor { // Creates or retrieve a Python descriptor of the specified type. // Objects are interned: the same descriptor will return the same object if it // was kept alive. +// 'was_created' is an optional pointer to a bool, and is set to true if a new +// object was allocated. // Always return a new reference. -PyObject* NewInternedDescriptor(PyTypeObject* type, const void* descriptor) { +PyObject* NewInternedDescriptor(PyTypeObject* type, const void* descriptor, + bool* was_created) { + if (was_created) { + *was_created = false; + } if (descriptor == NULL) { PyErr_BadInternalCall(); return NULL; @@ -283,6 +287,9 @@ PyObject* NewInternedDescriptor(PyTypeObject* type, const void* descriptor) { GetDescriptorPool()->interned_descriptors->insert( std::make_pair(descriptor, reinterpret_cast<PyObject*>(py_descriptor))); + if (was_created) { + *was_created = true; + } return reinterpret_cast<PyObject*>(py_descriptor); } @@ -298,9 +305,7 @@ static PyGetSetDef Getters[] = { PyTypeObject PyBaseDescriptor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - // Keep the fully qualified _message symbol in a line for opensource. - "google.protobuf.internal._message." - "DescriptorBase", // tp_name + FULL_MODULE_NAME ".DescriptorBase", // tp_name sizeof(PyBaseDescriptor), // tp_basicsize 0, // tp_itemsize (destructor)Dealloc, // tp_dealloc @@ -357,7 +362,7 @@ static PyObject* GetFullName(PyBaseDescriptor* self, void *closure) { } static PyObject* GetFile(PyBaseDescriptor *self, void *closure) { - return PyFileDescriptor_New(_GetDescriptor(self)->file()); + return PyFileDescriptor_FromDescriptor(_GetDescriptor(self)->file()); } static PyObject* GetConcreteClass(PyBaseDescriptor* self, void *closure) { @@ -367,17 +372,6 @@ static PyObject* GetConcreteClass(PyBaseDescriptor* self, void *closure) { return concrete_class; } -static int SetConcreteClass(PyBaseDescriptor *self, PyObject *value, - void *closure) { - // This attribute is also set from reflection.py. Check that it's actually a - // no-op. - if (value != cdescriptor_pool::GetMessageClass( - GetDescriptorPool(), _GetDescriptor(self))) { - PyErr_SetString(PyExc_AttributeError, "Cannot change _concrete_class"); - } - return 0; -} - static PyObject* GetFieldsByName(PyBaseDescriptor* self, void *closure) { return NewMessageFieldsByName(_GetDescriptor(self)); } @@ -452,7 +446,7 @@ static PyObject* GetContainingType(PyBaseDescriptor *self, void *closure) { const Descriptor* containing_type = _GetDescriptor(self)->containing_type(); if (containing_type) { - return PyMessageDescriptor_New(containing_type); + return PyMessageDescriptor_FromDescriptor(containing_type); } else { Py_RETURN_NONE; } @@ -515,29 +509,34 @@ static PyObject* GetSyntax(PyBaseDescriptor *self, void *closure) { } static PyGetSetDef Getters[] = { - { C("name"), (getter)GetName, NULL, "Last name", NULL}, - { C("full_name"), (getter)GetFullName, NULL, "Full name", NULL}, - { C("_concrete_class"), (getter)GetConcreteClass, (setter)SetConcreteClass, "concrete class", NULL}, - { C("file"), (getter)GetFile, NULL, "File descriptor", NULL}, - - { C("fields"), (getter)GetFieldsSeq, NULL, "Fields sequence", NULL}, - { C("fields_by_name"), (getter)GetFieldsByName, NULL, "Fields by name", NULL}, - { C("fields_by_number"), (getter)GetFieldsByNumber, NULL, "Fields by number", NULL}, - { C("nested_types"), (getter)GetNestedTypesSeq, NULL, "Nested types sequence", NULL}, - { C("nested_types_by_name"), (getter)GetNestedTypesByName, NULL, "Nested types by name", NULL}, - { C("extensions"), (getter)GetExtensions, NULL, "Extensions Sequence", NULL}, - { C("extensions_by_name"), (getter)GetExtensionsByName, NULL, "Extensions by name", NULL}, - { C("extension_ranges"), (getter)GetExtensionRanges, NULL, "Extension ranges", NULL}, - { C("enum_types"), (getter)GetEnumsSeq, NULL, "Enum sequence", NULL}, - { C("enum_types_by_name"), (getter)GetEnumTypesByName, NULL, "Enum types by name", NULL}, - { C("enum_values_by_name"), (getter)GetEnumValuesByName, NULL, "Enum values by name", NULL}, - { C("oneofs_by_name"), (getter)GetOneofsByName, NULL, "Oneofs by name", NULL}, - { C("oneofs"), (getter)GetOneofsSeq, NULL, "Oneofs by name", NULL}, - { C("containing_type"), (getter)GetContainingType, (setter)SetContainingType, "Containing type", NULL}, - { C("is_extendable"), (getter)IsExtendable, (setter)NULL, NULL, NULL}, - { C("has_options"), (getter)GetHasOptions, (setter)SetHasOptions, "Has Options", NULL}, - { C("_options"), (getter)NULL, (setter)SetOptions, "Options", NULL}, - { C("syntax"), (getter)GetSyntax, (setter)NULL, "Syntax", NULL}, + { "name", (getter)GetName, NULL, "Last name"}, + { "full_name", (getter)GetFullName, NULL, "Full name"}, + { "_concrete_class", (getter)GetConcreteClass, NULL, "concrete class"}, + { "file", (getter)GetFile, NULL, "File descriptor"}, + + { "fields", (getter)GetFieldsSeq, NULL, "Fields sequence"}, + { "fields_by_name", (getter)GetFieldsByName, NULL, "Fields by name"}, + { "fields_by_number", (getter)GetFieldsByNumber, NULL, "Fields by number"}, + { "nested_types", (getter)GetNestedTypesSeq, NULL, "Nested types sequence"}, + { "nested_types_by_name", (getter)GetNestedTypesByName, NULL, + "Nested types by name"}, + { "extensions", (getter)GetExtensions, NULL, "Extensions Sequence"}, + { "extensions_by_name", (getter)GetExtensionsByName, NULL, + "Extensions by name"}, + { "extension_ranges", (getter)GetExtensionRanges, NULL, "Extension ranges"}, + { "enum_types", (getter)GetEnumsSeq, NULL, "Enum sequence"}, + { "enum_types_by_name", (getter)GetEnumTypesByName, NULL, + "Enum types by name"}, + { "enum_values_by_name", (getter)GetEnumValuesByName, NULL, + "Enum values by name"}, + { "oneofs_by_name", (getter)GetOneofsByName, NULL, "Oneofs by name"}, + { "oneofs", (getter)GetOneofsSeq, NULL, "Oneofs by name"}, + { "containing_type", (getter)GetContainingType, (setter)SetContainingType, + "Containing type"}, + { "is_extendable", (getter)IsExtendable, (setter)NULL}, + { "has_options", (getter)GetHasOptions, (setter)SetHasOptions, "Has Options"}, + { "_options", (getter)NULL, (setter)SetOptions, "Options"}, + { "syntax", (getter)GetSyntax, (setter)NULL, "Syntax"}, {NULL} }; @@ -552,9 +551,7 @@ static PyMethodDef Methods[] = { PyTypeObject PyMessageDescriptor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - // Keep the fully qualified _message symbol in a line for opensource. - C("google.protobuf.internal._message." - "MessageDescriptor"), // tp_name + FULL_MODULE_NAME ".MessageDescriptor", // tp_name sizeof(PyBaseDescriptor), // tp_basicsize 0, // tp_itemsize 0, // tp_dealloc @@ -573,7 +570,7 @@ PyTypeObject PyMessageDescriptor_Type = { 0, // tp_setattro 0, // tp_as_buffer Py_TPFLAGS_DEFAULT, // tp_flags - C("A Message Descriptor"), // tp_doc + "A Message Descriptor", // tp_doc 0, // tp_traverse 0, // tp_clear 0, // tp_richcompare @@ -586,10 +583,10 @@ PyTypeObject PyMessageDescriptor_Type = { &descriptor::PyBaseDescriptor_Type, // tp_base }; -PyObject* PyMessageDescriptor_New( +PyObject* PyMessageDescriptor_FromDescriptor( const Descriptor* message_descriptor) { return descriptor::NewInternedDescriptor( - &PyMessageDescriptor_Type, message_descriptor); + &PyMessageDescriptor_Type, message_descriptor, NULL); } const Descriptor* PyMessageDescriptor_AsDescriptor(PyObject* obj) { @@ -715,7 +712,7 @@ static PyObject* GetCDescriptor(PyObject *self, void *closure) { static PyObject *GetEnumType(PyBaseDescriptor *self, void *closure) { const EnumDescriptor* enum_type = _GetDescriptor(self)->enum_type(); if (enum_type) { - return PyEnumDescriptor_New(enum_type); + return PyEnumDescriptor_FromDescriptor(enum_type); } else { Py_RETURN_NONE; } @@ -728,7 +725,7 @@ static int SetEnumType(PyBaseDescriptor *self, PyObject *value, void *closure) { static PyObject *GetMessageType(PyBaseDescriptor *self, void *closure) { const Descriptor* message_type = _GetDescriptor(self)->message_type(); if (message_type) { - return PyMessageDescriptor_New(message_type); + return PyMessageDescriptor_FromDescriptor(message_type); } else { Py_RETURN_NONE; } @@ -743,7 +740,7 @@ static PyObject* GetContainingType(PyBaseDescriptor *self, void *closure) { const Descriptor* containing_type = _GetDescriptor(self)->containing_type(); if (containing_type) { - return PyMessageDescriptor_New(containing_type); + return PyMessageDescriptor_FromDescriptor(containing_type); } else { Py_RETURN_NONE; } @@ -758,7 +755,7 @@ static PyObject* GetExtensionScope(PyBaseDescriptor *self, void *closure) { const Descriptor* extension_scope = _GetDescriptor(self)->extension_scope(); if (extension_scope) { - return PyMessageDescriptor_New(extension_scope); + return PyMessageDescriptor_FromDescriptor(extension_scope); } else { Py_RETURN_NONE; } @@ -768,7 +765,7 @@ static PyObject* GetContainingOneof(PyBaseDescriptor *self, void *closure) { const OneofDescriptor* containing_oneof = _GetDescriptor(self)->containing_oneof(); if (containing_oneof) { - return PyOneofDescriptor_New(containing_oneof); + return PyOneofDescriptor_FromDescriptor(containing_oneof); } else { Py_RETURN_NONE; } @@ -803,26 +800,30 @@ static int SetOptions(PyBaseDescriptor *self, PyObject *value, static PyGetSetDef Getters[] = { - { C("full_name"), (getter)GetFullName, NULL, "Full name", NULL}, - { C("name"), (getter)GetName, NULL, "Unqualified name", NULL}, - { C("type"), (getter)GetType, NULL, "C++ Type", NULL}, - { C("cpp_type"), (getter)GetCppType, NULL, "C++ Type", NULL}, - { C("label"), (getter)GetLabel, NULL, "Label", NULL}, - { C("number"), (getter)GetNumber, NULL, "Number", NULL}, - { C("index"), (getter)GetIndex, NULL, "Index", NULL}, - { C("default_value"), (getter)GetDefaultValue, NULL, "Default Value", NULL}, - { C("has_default_value"), (getter)HasDefaultValue, NULL, NULL, NULL}, - { C("is_extension"), (getter)IsExtension, NULL, "ID", NULL}, - { C("id"), (getter)GetID, NULL, "ID", NULL}, - { C("_cdescriptor"), (getter)GetCDescriptor, NULL, "HAACK REMOVE ME", NULL}, - - { C("message_type"), (getter)GetMessageType, (setter)SetMessageType, "Message type", NULL}, - { C("enum_type"), (getter)GetEnumType, (setter)SetEnumType, "Enum type", NULL}, - { C("containing_type"), (getter)GetContainingType, (setter)SetContainingType, "Containing type", NULL}, - { C("extension_scope"), (getter)GetExtensionScope, (setter)NULL, "Extension scope", NULL}, - { C("containing_oneof"), (getter)GetContainingOneof, (setter)SetContainingOneof, "Containing oneof", NULL}, - { C("has_options"), (getter)GetHasOptions, (setter)SetHasOptions, "Has Options", NULL}, - { C("_options"), (getter)NULL, (setter)SetOptions, "Options", NULL}, + { "full_name", (getter)GetFullName, NULL, "Full name"}, + { "name", (getter)GetName, NULL, "Unqualified name"}, + { "type", (getter)GetType, NULL, "C++ Type"}, + { "cpp_type", (getter)GetCppType, NULL, "C++ Type"}, + { "label", (getter)GetLabel, NULL, "Label"}, + { "number", (getter)GetNumber, NULL, "Number"}, + { "index", (getter)GetIndex, NULL, "Index"}, + { "default_value", (getter)GetDefaultValue, NULL, "Default Value"}, + { "has_default_value", (getter)HasDefaultValue}, + { "is_extension", (getter)IsExtension, NULL, "ID"}, + { "id", (getter)GetID, NULL, "ID"}, + { "_cdescriptor", (getter)GetCDescriptor, NULL, "HAACK REMOVE ME"}, + + { "message_type", (getter)GetMessageType, (setter)SetMessageType, + "Message type"}, + { "enum_type", (getter)GetEnumType, (setter)SetEnumType, "Enum type"}, + { "containing_type", (getter)GetContainingType, (setter)SetContainingType, + "Containing type"}, + { "extension_scope", (getter)GetExtensionScope, (setter)NULL, + "Extension scope"}, + { "containing_oneof", (getter)GetContainingOneof, (setter)SetContainingOneof, + "Containing oneof"}, + { "has_options", (getter)GetHasOptions, (setter)SetHasOptions, "Has Options"}, + { "_options", (getter)NULL, (setter)SetOptions, "Options"}, {NULL} }; @@ -835,8 +836,7 @@ static PyMethodDef Methods[] = { PyTypeObject PyFieldDescriptor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - C("google.protobuf.internal." - "_message.FieldDescriptor"), // tp_name + FULL_MODULE_NAME ".FieldDescriptor", // tp_name sizeof(PyBaseDescriptor), // tp_basicsize 0, // tp_itemsize 0, // tp_dealloc @@ -855,7 +855,7 @@ PyTypeObject PyFieldDescriptor_Type = { 0, // tp_setattro 0, // tp_as_buffer Py_TPFLAGS_DEFAULT, // tp_flags - C("A Field Descriptor"), // tp_doc + "A Field Descriptor", // tp_doc 0, // tp_traverse 0, // tp_clear 0, // tp_richcompare @@ -868,10 +868,10 @@ PyTypeObject PyFieldDescriptor_Type = { &descriptor::PyBaseDescriptor_Type, // tp_base }; -PyObject* PyFieldDescriptor_New( +PyObject* PyFieldDescriptor_FromDescriptor( const FieldDescriptor* field_descriptor) { return descriptor::NewInternedDescriptor( - &PyFieldDescriptor_Type, field_descriptor); + &PyFieldDescriptor_Type, field_descriptor, NULL); } const FieldDescriptor* PyFieldDescriptor_AsDescriptor(PyObject* obj) { @@ -900,7 +900,7 @@ static PyObject* GetName(PyBaseDescriptor *self, void *closure) { } static PyObject* GetFile(PyBaseDescriptor *self, void *closure) { - return PyFileDescriptor_New(_GetDescriptor(self)->file()); + return PyFileDescriptor_FromDescriptor(_GetDescriptor(self)->file()); } static PyObject* GetEnumvaluesByName(PyBaseDescriptor* self, void *closure) { @@ -919,7 +919,7 @@ static PyObject* GetContainingType(PyBaseDescriptor *self, void *closure) { const Descriptor* containing_type = _GetDescriptor(self)->containing_type(); if (containing_type) { - return PyMessageDescriptor_New(containing_type); + return PyMessageDescriptor_FromDescriptor(containing_type); } else { Py_RETURN_NONE; } @@ -964,16 +964,19 @@ static PyMethodDef Methods[] = { }; static PyGetSetDef Getters[] = { - { C("full_name"), (getter)GetFullName, NULL, "Full name", NULL}, - { C("name"), (getter)GetName, NULL, "last name", NULL}, - { C("file"), (getter)GetFile, NULL, "File descriptor", NULL}, - { C("values"), (getter)GetEnumvaluesSeq, NULL, "values", NULL}, - { C("values_by_name"), (getter)GetEnumvaluesByName, NULL, "Enumvalues by name", NULL}, - { C("values_by_number"), (getter)GetEnumvaluesByNumber, NULL, "Enumvalues by number", NULL}, - - { C("containing_type"), (getter)GetContainingType, (setter)SetContainingType, "Containing type", NULL}, - { C("has_options"), (getter)GetHasOptions, (setter)SetHasOptions, "Has Options", NULL}, - { C("_options"), (getter)NULL, (setter)SetOptions, "Options", NULL}, + { "full_name", (getter)GetFullName, NULL, "Full name"}, + { "name", (getter)GetName, NULL, "last name"}, + { "file", (getter)GetFile, NULL, "File descriptor"}, + { "values", (getter)GetEnumvaluesSeq, NULL, "values"}, + { "values_by_name", (getter)GetEnumvaluesByName, NULL, + "Enum values by name"}, + { "values_by_number", (getter)GetEnumvaluesByNumber, NULL, + "Enum values by number"}, + + { "containing_type", (getter)GetContainingType, (setter)SetContainingType, + "Containing type"}, + { "has_options", (getter)GetHasOptions, (setter)SetHasOptions, "Has Options"}, + { "_options", (getter)NULL, (setter)SetOptions, "Options"}, {NULL} }; @@ -981,9 +984,7 @@ static PyGetSetDef Getters[] = { PyTypeObject PyEnumDescriptor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - // Keep the fully qualified _message symbol in a line for opensource. - C("google.protobuf.internal._message." - "EnumDescriptor"), // tp_name + FULL_MODULE_NAME ".EnumDescriptor", // tp_name sizeof(PyBaseDescriptor), // tp_basicsize 0, // tp_itemsize 0, // tp_dealloc @@ -1002,7 +1003,7 @@ PyTypeObject PyEnumDescriptor_Type = { 0, // tp_setattro 0, // tp_as_buffer Py_TPFLAGS_DEFAULT, // tp_flags - C("A Enum Descriptor"), // tp_doc + "A Enum Descriptor", // tp_doc 0, // tp_traverse 0, // tp_clear 0, // tp_richcompare @@ -1015,10 +1016,10 @@ PyTypeObject PyEnumDescriptor_Type = { &descriptor::PyBaseDescriptor_Type, // tp_base }; -PyObject* PyEnumDescriptor_New( +PyObject* PyEnumDescriptor_FromDescriptor( const EnumDescriptor* enum_descriptor) { return descriptor::NewInternedDescriptor( - &PyEnumDescriptor_Type, enum_descriptor); + &PyEnumDescriptor_Type, enum_descriptor, NULL); } namespace enumvalue_descriptor { @@ -1042,7 +1043,7 @@ static PyObject* GetIndex(PyBaseDescriptor *self, void *closure) { } static PyObject* GetType(PyBaseDescriptor *self, void *closure) { - return PyEnumDescriptor_New(_GetDescriptor(self)->type()); + return PyEnumDescriptor_FromDescriptor(_GetDescriptor(self)->type()); } static PyObject* GetHasOptions(PyBaseDescriptor *self, void *closure) { @@ -1069,13 +1070,13 @@ static int SetOptions(PyBaseDescriptor *self, PyObject *value, static PyGetSetDef Getters[] = { - { C("name"), (getter)GetName, NULL, "name", NULL}, - { C("number"), (getter)GetNumber, NULL, "number", NULL}, - { C("index"), (getter)GetIndex, NULL, "index", NULL}, - { C("type"), (getter)GetType, NULL, "index", NULL}, + { "name", (getter)GetName, NULL, "name"}, + { "number", (getter)GetNumber, NULL, "number"}, + { "index", (getter)GetIndex, NULL, "index"}, + { "type", (getter)GetType, NULL, "index"}, - { C("has_options"), (getter)GetHasOptions, (setter)SetHasOptions, "Has Options", NULL}, - { C("_options"), (getter)NULL, (setter)SetOptions, "Options", NULL}, + { "has_options", (getter)GetHasOptions, (setter)SetHasOptions, "Has Options"}, + { "_options", (getter)NULL, (setter)SetOptions, "Options"}, {NULL} }; @@ -1088,8 +1089,7 @@ static PyMethodDef Methods[] = { PyTypeObject PyEnumValueDescriptor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - C("google.protobuf.internal." - "_message.EnumValueDescriptor"), // tp_name + FULL_MODULE_NAME ".EnumValueDescriptor", // tp_name sizeof(PyBaseDescriptor), // tp_basicsize 0, // tp_itemsize 0, // tp_dealloc @@ -1108,7 +1108,7 @@ PyTypeObject PyEnumValueDescriptor_Type = { 0, // tp_setattro 0, // tp_as_buffer Py_TPFLAGS_DEFAULT, // tp_flags - C("A EnumValue Descriptor"), // tp_doc + "A EnumValue Descriptor", // tp_doc 0, // tp_traverse 0, // tp_clear 0, // tp_richcompare @@ -1121,10 +1121,10 @@ PyTypeObject PyEnumValueDescriptor_Type = { &descriptor::PyBaseDescriptor_Type, // tp_base }; -PyObject* PyEnumValueDescriptor_New( +PyObject* PyEnumValueDescriptor_FromDescriptor( const EnumValueDescriptor* enumvalue_descriptor) { return descriptor::NewInternedDescriptor( - &PyEnumValueDescriptor_Type, enumvalue_descriptor); + &PyEnumValueDescriptor_Type, enumvalue_descriptor, NULL); } namespace file_descriptor { @@ -1218,18 +1218,20 @@ static PyObject* CopyToProto(PyFileDescriptor *self, PyObject *target) { } static PyGetSetDef Getters[] = { - { C("name"), (getter)GetName, NULL, "name", NULL}, - { C("package"), (getter)GetPackage, NULL, "package", NULL}, - { C("serialized_pb"), (getter)GetSerializedPb, NULL, NULL, NULL}, - { C("message_types_by_name"), (getter)GetMessageTypesByName, NULL, "Messages by name", NULL}, - { C("enum_types_by_name"), (getter)GetEnumTypesByName, NULL, "Enums by name", NULL}, - { C("extensions_by_name"), (getter)GetExtensionsByName, NULL, "Extensions by name", NULL}, - { C("dependencies"), (getter)GetDependencies, NULL, "Dependencies", NULL}, - { C("public_dependencies"), (getter)GetPublicDependencies, NULL, "Dependencies", NULL}, - - { C("has_options"), (getter)GetHasOptions, (setter)SetHasOptions, "Has Options", NULL}, - { C("_options"), (getter)NULL, (setter)SetOptions, "Options", NULL}, - { C("syntax"), (getter)GetSyntax, (setter)NULL, "Syntax", NULL}, + { "name", (getter)GetName, NULL, "name"}, + { "package", (getter)GetPackage, NULL, "package"}, + { "serialized_pb", (getter)GetSerializedPb}, + { "message_types_by_name", (getter)GetMessageTypesByName, NULL, + "Messages by name"}, + { "enum_types_by_name", (getter)GetEnumTypesByName, NULL, "Enums by name"}, + { "extensions_by_name", (getter)GetExtensionsByName, NULL, + "Extensions by name"}, + { "dependencies", (getter)GetDependencies, NULL, "Dependencies"}, + { "public_dependencies", (getter)GetPublicDependencies, NULL, "Dependencies"}, + + { "has_options", (getter)GetHasOptions, (setter)SetHasOptions, "Has Options"}, + { "_options", (getter)NULL, (setter)SetOptions, "Options"}, + { "syntax", (getter)GetSyntax, (setter)NULL, "Syntax"}, {NULL} }; @@ -1243,11 +1245,10 @@ static PyMethodDef Methods[] = { PyTypeObject PyFileDescriptor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - C("google.protobuf.internal." - "_message.FileDescriptor"), // tp_name + FULL_MODULE_NAME ".FileDescriptor", // tp_name sizeof(PyFileDescriptor), // tp_basicsize 0, // tp_itemsize - (destructor)file_descriptor::Dealloc, // tp_dealloc + (destructor)file_descriptor::Dealloc, // tp_dealloc 0, // tp_print 0, // tp_getattr 0, // tp_setattr @@ -1263,7 +1264,7 @@ PyTypeObject PyFileDescriptor_Type = { 0, // tp_setattro 0, // tp_as_buffer Py_TPFLAGS_DEFAULT, // tp_flags - C("A File Descriptor"), // tp_doc + "A File Descriptor", // tp_doc 0, // tp_traverse 0, // tp_clear 0, // tp_richcompare @@ -1284,23 +1285,28 @@ PyTypeObject PyFileDescriptor_Type = { PyObject_Del, // tp_free }; -PyObject* PyFileDescriptor_New(const FileDescriptor* file_descriptor) { - return descriptor::NewInternedDescriptor( - &PyFileDescriptor_Type, file_descriptor); +PyObject* PyFileDescriptor_FromDescriptor( + const FileDescriptor* file_descriptor) { + return PyFileDescriptor_FromDescriptorWithSerializedPb(file_descriptor, + NULL); } -PyObject* PyFileDescriptor_NewWithPb( +PyObject* PyFileDescriptor_FromDescriptorWithSerializedPb( const FileDescriptor* file_descriptor, PyObject *serialized_pb) { - PyObject* py_descriptor = PyFileDescriptor_New(file_descriptor); + bool was_created; + PyObject* py_descriptor = descriptor::NewInternedDescriptor( + &PyFileDescriptor_Type, file_descriptor, &was_created); if (py_descriptor == NULL) { return NULL; } - if (serialized_pb != NULL) { + if (was_created) { PyFileDescriptor* cfile_descriptor = reinterpret_cast<PyFileDescriptor*>(py_descriptor); Py_XINCREF(serialized_pb); cfile_descriptor->serialized_pb = serialized_pb; } + // TODO(amauryfa): In the case of a cached object, check that serialized_pb + // is the same as before. return py_descriptor; } @@ -1333,19 +1339,19 @@ static PyObject* GetContainingType(PyBaseDescriptor *self, void *closure) { const Descriptor* containing_type = _GetDescriptor(self)->containing_type(); if (containing_type) { - return PyMessageDescriptor_New(containing_type); + return PyMessageDescriptor_FromDescriptor(containing_type); } else { Py_RETURN_NONE; } } static PyGetSetDef Getters[] = { - { C("name"), (getter)GetName, NULL, "Name", NULL}, - { C("full_name"), (getter)GetFullName, NULL, "Full name", NULL}, - { C("index"), (getter)GetIndex, NULL, "Index", NULL}, + { "name", (getter)GetName, NULL, "Name"}, + { "full_name", (getter)GetFullName, NULL, "Full name"}, + { "index", (getter)GetIndex, NULL, "Index"}, - { C("containing_type"), (getter)GetContainingType, NULL, "Containing type", NULL}, - { C("fields"), (getter)GetFields, NULL, "Fields", NULL}, + { "containing_type", (getter)GetContainingType, NULL, "Containing type"}, + { "fields", (getter)GetFields, NULL, "Fields"}, {NULL} }; @@ -1353,8 +1359,7 @@ static PyGetSetDef Getters[] = { PyTypeObject PyOneofDescriptor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - C("google.protobuf.internal." - "_message.OneofDescriptor"), // tp_name + FULL_MODULE_NAME ".OneofDescriptor", // tp_name sizeof(PyBaseDescriptor), // tp_basicsize 0, // tp_itemsize 0, // tp_dealloc @@ -1373,7 +1378,7 @@ PyTypeObject PyOneofDescriptor_Type = { 0, // tp_setattro 0, // tp_as_buffer Py_TPFLAGS_DEFAULT, // tp_flags - C("A Oneof Descriptor"), // tp_doc + "A Oneof Descriptor", // tp_doc 0, // tp_traverse 0, // tp_clear 0, // tp_richcompare @@ -1386,10 +1391,10 @@ PyTypeObject PyOneofDescriptor_Type = { &descriptor::PyBaseDescriptor_Type, // tp_base }; -PyObject* PyOneofDescriptor_New( +PyObject* PyOneofDescriptor_FromDescriptor( const OneofDescriptor* oneof_descriptor) { return descriptor::NewInternedDescriptor( - &PyOneofDescriptor_Type, oneof_descriptor); + &PyOneofDescriptor_Type, oneof_descriptor, NULL); } // Add a enum values to a type dictionary. diff --git a/python/google/protobuf/pyext/descriptor.h b/python/google/protobuf/pyext/descriptor.h index ba6e7298..b2550406 100644 --- a/python/google/protobuf/pyext/descriptor.h +++ b/python/google/protobuf/pyext/descriptor.h @@ -48,21 +48,24 @@ extern PyTypeObject PyEnumValueDescriptor_Type; extern PyTypeObject PyFileDescriptor_Type; extern PyTypeObject PyOneofDescriptor_Type; -// Return a new reference to a Descriptor object. +// Wraps a Descriptor in a Python object. // The C++ pointer is usually borrowed from the global DescriptorPool. // In any case, it must stay alive as long as the Python object. -PyObject* PyMessageDescriptor_New(const Descriptor* descriptor); -PyObject* PyFieldDescriptor_New(const FieldDescriptor* descriptor); -PyObject* PyEnumDescriptor_New(const EnumDescriptor* descriptor); -PyObject* PyEnumValueDescriptor_New(const EnumValueDescriptor* descriptor); -PyObject* PyOneofDescriptor_New(const OneofDescriptor* descriptor); -PyObject* PyFileDescriptor_New(const FileDescriptor* file_descriptor); +// Returns a new reference. +PyObject* PyMessageDescriptor_FromDescriptor(const Descriptor* descriptor); +PyObject* PyFieldDescriptor_FromDescriptor(const FieldDescriptor* descriptor); +PyObject* PyEnumDescriptor_FromDescriptor(const EnumDescriptor* descriptor); +PyObject* PyEnumValueDescriptor_FromDescriptor( + const EnumValueDescriptor* descriptor); +PyObject* PyOneofDescriptor_FromDescriptor(const OneofDescriptor* descriptor); +PyObject* PyFileDescriptor_FromDescriptor( + const FileDescriptor* file_descriptor); // Alternate constructor of PyFileDescriptor, used when we already have a // serialized FileDescriptorProto that can be cached. // Returns a new reference. -PyObject* PyFileDescriptor_NewWithPb(const FileDescriptor* file_descriptor, - PyObject* serialized_pb); +PyObject* PyFileDescriptor_FromDescriptorWithSerializedPb( + const FileDescriptor* file_descriptor, PyObject* serialized_pb); // Return the C++ descriptor pointer. // This function checks the parameter type; on error, return NULL with a Python diff --git a/python/google/protobuf/pyext/descriptor_containers.cc b/python/google/protobuf/pyext/descriptor_containers.cc index 06edebf8..92e11e31 100644 --- a/python/google/protobuf/pyext/descriptor_containers.cc +++ b/python/google/protobuf/pyext/descriptor_containers.cc @@ -898,7 +898,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) { } static PyObject* NewObjectFromItem(ItemDescriptor item) { - return PyFieldDescriptor_New(item); + return PyFieldDescriptor_FromDescriptor(item); } static const string& GetItemName(ItemDescriptor item) { @@ -956,7 +956,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) { } static PyObject* NewObjectFromItem(ItemDescriptor item) { - return PyMessageDescriptor_New(item); + return PyMessageDescriptor_FromDescriptor(item); } static const string& GetItemName(ItemDescriptor item) { @@ -1006,7 +1006,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) { } static PyObject* NewObjectFromItem(ItemDescriptor item) { - return PyEnumDescriptor_New(item); + return PyEnumDescriptor_FromDescriptor(item); } static const string& GetItemName(ItemDescriptor item) { @@ -1082,7 +1082,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) { } static PyObject* NewObjectFromItem(ItemDescriptor item) { - return PyEnumValueDescriptor_New(item); + return PyEnumValueDescriptor_FromDescriptor(item); } static const string& GetItemName(ItemDescriptor item) { @@ -1124,7 +1124,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) { } static PyObject* NewObjectFromItem(ItemDescriptor item) { - return PyFieldDescriptor_New(item); + return PyFieldDescriptor_FromDescriptor(item); } static const string& GetItemName(ItemDescriptor item) { @@ -1174,7 +1174,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) { } static PyObject* NewObjectFromItem(ItemDescriptor item) { - return PyOneofDescriptor_New(item); + return PyOneofDescriptor_FromDescriptor(item); } static const string& GetItemName(ItemDescriptor item) { @@ -1238,7 +1238,7 @@ static ItemDescriptor GetByNumber(PyContainer* self, int number) { } static PyObject* NewObjectFromItem(ItemDescriptor item) { - return PyEnumValueDescriptor_New(item); + return PyEnumValueDescriptor_FromDescriptor(item); } static const string& GetItemName(ItemDescriptor item) { @@ -1302,7 +1302,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) { } static PyObject* NewObjectFromItem(ItemDescriptor item) { - return PyFieldDescriptor_New(item); + return PyFieldDescriptor_FromDescriptor(item); } static int GetItemIndex(ItemDescriptor item) { @@ -1354,7 +1354,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) { } static PyObject* NewObjectFromItem(ItemDescriptor item) { - return PyMessageDescriptor_New(item); + return PyMessageDescriptor_FromDescriptor(item); } static const string& GetItemName(ItemDescriptor item) { @@ -1400,7 +1400,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) { } static PyObject* NewObjectFromItem(ItemDescriptor item) { - return PyEnumDescriptor_New(item); + return PyEnumDescriptor_FromDescriptor(item); } static const string& GetItemName(ItemDescriptor item) { @@ -1446,7 +1446,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) { } static PyObject* NewObjectFromItem(ItemDescriptor item) { - return PyFieldDescriptor_New(item); + return PyFieldDescriptor_FromDescriptor(item); } static const string& GetItemName(ItemDescriptor item) { @@ -1488,7 +1488,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) { } static PyObject* NewObjectFromItem(ItemDescriptor item) { - return PyFileDescriptor_New(item); + return PyFileDescriptor_FromDescriptor(item); } static DescriptorContainerDef ContainerDef = { @@ -1522,7 +1522,7 @@ static ItemDescriptor GetByIndex(PyContainer* self, int index) { } static PyObject* NewObjectFromItem(ItemDescriptor item) { - return PyFileDescriptor_New(item); + return PyFileDescriptor_FromDescriptor(item); } static DescriptorContainerDef ContainerDef = { diff --git a/python/google/protobuf/pyext/descriptor_containers.h b/python/google/protobuf/pyext/descriptor_containers.h index d81537de..8fbdaff9 100644 --- a/python/google/protobuf/pyext/descriptor_containers.h +++ b/python/google/protobuf/pyext/descriptor_containers.h @@ -28,6 +28,9 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_CONTAINERS_H__ +#define GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_CONTAINERS_H__ + // Mappings and Sequences of descriptors. // They implement containers like fields_by_name, EnumDescriptor.values... // See descriptor_containers.cc for more description. @@ -92,4 +95,6 @@ PyObject* NewFilePublicDependencies(const FileDescriptor* descriptor); } // namespace python } // namespace protobuf + } // namespace google +#endif // GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_CONTAINERS_H__ diff --git a/python/google/protobuf/pyext/descriptor_pool.cc b/python/google/protobuf/pyext/descriptor_pool.cc index bc3077bc..ecd90847 100644 --- a/python/google/protobuf/pyext/descriptor_pool.cc +++ b/python/google/protobuf/pyext/descriptor_pool.cc @@ -35,10 +35,9 @@ #include <google/protobuf/descriptor.pb.h> #include <google/protobuf/pyext/descriptor_pool.h> #include <google/protobuf/pyext/descriptor.h> +#include <google/protobuf/pyext/message.h> #include <google/protobuf/pyext/scoped_pyobject_ptr.h> -#define C(str) const_cast<char*>(str) - #if PY_MAJOR_VERSION >= 3 #define PyString_FromStringAndSize PyUnicode_FromStringAndSize #if PY_VERSION_HEX < 0x03030000 @@ -108,11 +107,11 @@ PyObject* FindMessageByName(PyDescriptorPool* self, PyObject* arg) { self->pool->FindMessageTypeByName(string(name, name_size)); if (message_descriptor == NULL) { - PyErr_Format(PyExc_TypeError, "Couldn't find message %.200s", name); + PyErr_Format(PyExc_KeyError, "Couldn't find message %.200s", name); return NULL; } - return PyMessageDescriptor_New(message_descriptor); + return PyMessageDescriptor_FromDescriptor(message_descriptor); } // Add a message class to our database. @@ -158,6 +157,24 @@ PyObject *GetMessageClass(PyDescriptorPool* self, } } +PyObject* FindFileByName(PyDescriptorPool* self, PyObject* arg) { + Py_ssize_t name_size; + char* name; + if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) { + return NULL; + } + + const FileDescriptor* file_descriptor = + self->pool->FindFileByName(string(name, name_size)); + if (file_descriptor == NULL) { + PyErr_Format(PyExc_KeyError, "Couldn't find file %.200s", + name); + return NULL; + } + + return PyFileDescriptor_FromDescriptor(file_descriptor); +} + PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* arg) { Py_ssize_t name_size; char* name; @@ -168,12 +185,12 @@ PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* arg) { const FieldDescriptor* field_descriptor = self->pool->FindFieldByName(string(name, name_size)); if (field_descriptor == NULL) { - PyErr_Format(PyExc_TypeError, "Couldn't find field %.200s", + PyErr_Format(PyExc_KeyError, "Couldn't find field %.200s", name); return NULL; } - return PyFieldDescriptor_New(field_descriptor); + return PyFieldDescriptor_FromDescriptor(field_descriptor); } PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg) { @@ -186,11 +203,11 @@ PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg) { const FieldDescriptor* field_descriptor = self->pool->FindExtensionByName(string(name, name_size)); if (field_descriptor == NULL) { - PyErr_Format(PyExc_TypeError, "Couldn't find field %.200s", name); + PyErr_Format(PyExc_KeyError, "Couldn't find extension field %.200s", name); return NULL; } - return PyFieldDescriptor_New(field_descriptor); + return PyFieldDescriptor_FromDescriptor(field_descriptor); } PyObject* FindEnumTypeByName(PyDescriptorPool* self, PyObject* arg) { @@ -203,11 +220,11 @@ PyObject* FindEnumTypeByName(PyDescriptorPool* self, PyObject* arg) { const EnumDescriptor* enum_descriptor = self->pool->FindEnumTypeByName(string(name, name_size)); if (enum_descriptor == NULL) { - PyErr_Format(PyExc_TypeError, "Couldn't find enum %.200s", name); + PyErr_Format(PyExc_KeyError, "Couldn't find enum %.200s", name); return NULL; } - return PyEnumDescriptor_New(enum_descriptor); + return PyEnumDescriptor_FromDescriptor(enum_descriptor); } PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg) { @@ -220,70 +237,13 @@ PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg) { const OneofDescriptor* oneof_descriptor = self->pool->FindOneofByName(string(name, name_size)); if (oneof_descriptor == NULL) { - PyErr_Format(PyExc_TypeError, "Couldn't find oneof %.200s", name); + PyErr_Format(PyExc_KeyError, "Couldn't find oneof %.200s", name); return NULL; } - return PyOneofDescriptor_New(oneof_descriptor); + return PyOneofDescriptor_FromDescriptor(oneof_descriptor); } -static PyMethodDef Methods[] = { - { C("FindFieldByName"), - (PyCFunction)FindFieldByName, - METH_O, - C("Searches for a field descriptor by full name.") }, - { C("FindExtensionByName"), - (PyCFunction)FindExtensionByName, - METH_O, - C("Searches for extension descriptor by full name.") }, - {NULL} -}; - -} // namespace cdescriptor_pool - -PyTypeObject PyDescriptorPool_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - C("google.protobuf.internal." - "_message.DescriptorPool"), // tp_name - sizeof(PyDescriptorPool), // tp_basicsize - 0, // tp_itemsize - (destructor)cdescriptor_pool::Dealloc, // tp_dealloc - 0, // tp_print - 0, // tp_getattr - 0, // tp_setattr - 0, // tp_compare - 0, // tp_repr - 0, // tp_as_number - 0, // tp_as_sequence - 0, // tp_as_mapping - 0, // tp_hash - 0, // tp_call - 0, // tp_str - 0, // tp_getattro - 0, // tp_setattro - 0, // tp_as_buffer - Py_TPFLAGS_DEFAULT, // tp_flags - C("A Descriptor Pool"), // tp_doc - 0, // tp_traverse - 0, // tp_clear - 0, // tp_richcompare - 0, // tp_weaklistoffset - 0, // tp_iter - 0, // tp_iternext - cdescriptor_pool::Methods, // tp_methods - 0, // tp_members - 0, // tp_getset - 0, // tp_base - 0, // tp_dict - 0, // tp_descr_get - 0, // tp_descr_set - 0, // tp_dictoffset - 0, // tp_init - 0, // tp_alloc - 0, // tp_new - PyObject_Del, // tp_free -}; - // The code below loads new Descriptors from a serialized FileDescriptorProto. @@ -301,6 +261,7 @@ class BuildFileErrorCollector : public DescriptorPool::ErrorCollector { if (!had_errors) { error_message += ("Invalid proto descriptor for file \"" + filename + "\":\n"); + had_errors = true; } // As this only happens on failure and will result in the program not // running at all, no effort is made to optimize this string manipulation. @@ -311,7 +272,7 @@ class BuildFileErrorCollector : public DescriptorPool::ErrorCollector { bool had_errors; }; -PyObject* Python_BuildFile(PyObject* ignored, PyObject* serialized_pb) { +PyObject* AddSerializedFile(PyDescriptorPool* self, PyObject* serialized_pb) { char* message_type; Py_ssize_t message_len; @@ -330,13 +291,14 @@ PyObject* Python_BuildFile(PyObject* ignored, PyObject* serialized_pb) { const FileDescriptor* generated_file = DescriptorPool::generated_pool()->FindFileByName(file_proto.name()); if (generated_file != NULL) { - return PyFileDescriptor_NewWithPb(generated_file, serialized_pb); + return PyFileDescriptor_FromDescriptorWithSerializedPb( + generated_file, serialized_pb); } BuildFileErrorCollector error_collector; const FileDescriptor* descriptor = - GetDescriptorPool()->pool->BuildFileCollectingErrors(file_proto, - &error_collector); + self->pool->BuildFileCollectingErrors(file_proto, + &error_collector); if (descriptor == NULL) { PyErr_Format(PyExc_TypeError, "Couldn't build proto file into descriptor pool!\n%s", @@ -344,9 +306,84 @@ PyObject* Python_BuildFile(PyObject* ignored, PyObject* serialized_pb) { return NULL; } - return PyFileDescriptor_NewWithPb(descriptor, serialized_pb); + return PyFileDescriptor_FromDescriptorWithSerializedPb( + descriptor, serialized_pb); +} + +PyObject* Add(PyDescriptorPool* self, PyObject* file_descriptor_proto) { + ScopedPyObjectPtr serialized_pb( + PyObject_CallMethod(file_descriptor_proto, "SerializeToString", NULL)); + if (serialized_pb == NULL) { + return NULL; + } + return AddSerializedFile(self, serialized_pb); } +static PyMethodDef Methods[] = { + { "Add", (PyCFunction)Add, METH_O, + "Adds the FileDescriptorProto and its types to this pool." }, + { "AddSerializedFile", (PyCFunction)AddSerializedFile, METH_O, + "Adds a serialized FileDescriptorProto to this pool." }, + + { "FindFileByName", (PyCFunction)FindFileByName, METH_O, + "Searches for a file descriptor by its .proto name." }, + { "FindMessageTypeByName", (PyCFunction)FindMessageByName, METH_O, + "Searches for a message descriptor by full name." }, + { "FindFieldByName", (PyCFunction)FindFieldByName, METH_O, + "Searches for a field descriptor by full name." }, + { "FindExtensionByName", (PyCFunction)FindExtensionByName, METH_O, + "Searches for extension descriptor by full name." }, + { "FindEnumTypeByName", (PyCFunction)FindEnumTypeByName, METH_O, + "Searches for enum type descriptor by full name." }, + { "FindOneofByName", (PyCFunction)FindOneofByName, METH_O, + "Searches for oneof descriptor by full name." }, + {NULL} +}; + +} // namespace cdescriptor_pool + +PyTypeObject PyDescriptorPool_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + FULL_MODULE_NAME ".DescriptorPool", // tp_name + sizeof(PyDescriptorPool), // tp_basicsize + 0, // tp_itemsize + (destructor)cdescriptor_pool::Dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "A Descriptor Pool", // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + cdescriptor_pool::Methods, // tp_methods + 0, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init + 0, // tp_alloc + 0, // tp_new + PyObject_Del, // tp_free +}; + static PyDescriptorPool* global_cdescriptor_pool = NULL; bool InitDescriptorPool() { diff --git a/python/google/protobuf/pyext/descriptor_pool.h b/python/google/protobuf/pyext/descriptor_pool.h index 4e494b89..efb1abeb 100644 --- a/python/google/protobuf/pyext/descriptor_pool.h +++ b/python/google/protobuf/pyext/descriptor_pool.h @@ -95,6 +95,8 @@ const Descriptor* FindMessageTypeByName(PyDescriptorPool* self, const Descriptor* RegisterMessageClass( PyDescriptorPool* self, PyObject* message_class, PyObject* descriptor); +// The function below are also exposed as methods of the DescriptorPool type. + // Retrieves the Python class registered with the given message descriptor. // // Returns a *borrowed* reference if found, otherwise returns NULL with an @@ -134,12 +136,8 @@ PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg); } // namespace cdescriptor_pool -// Implement the Python "_BuildFile" method, it takes a serialized -// FileDescriptorProto, and adds it to the C++ DescriptorPool. -// It returns a new FileDescriptor object, or NULL when an exception is raised. -PyObject* Python_BuildFile(PyObject* ignored, PyObject* args); - // Retrieve the global descriptor pool owned by the _message module. +// Returns a *borrowed* reference. PyDescriptorPool* GetDescriptorPool(); // Initialize objects used by this module. diff --git a/python/google/protobuf/pyext/extension_dict.cc b/python/google/protobuf/pyext/extension_dict.cc index 8e38fc42..b8d18f8d 100644 --- a/python/google/protobuf/pyext/extension_dict.cc +++ b/python/google/protobuf/pyext/extension_dict.cc @@ -51,16 +51,6 @@ namespace python { namespace extension_dict { -// TODO(tibell): Always use self->message for clarity, just like in -// RepeatedCompositeContainer. -static Message* GetMessage(ExtensionDict* self) { - if (self->parent != NULL) { - return self->parent->message; - } else { - return self->message; - } -} - PyObject* len(ExtensionDict* self) { #if PY_MAJOR_VERSION >= 3 return PyLong_FromLong(PyDict_Size(self->values)); @@ -89,7 +79,7 @@ int ReleaseExtension(ExtensionDict* self, } } else if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { if (cmessage::ReleaseSubMessage( - GetMessage(self), descriptor, + self->parent, descriptor, reinterpret_cast<CMessage*>(extension)) < 0) { return -1; } @@ -109,7 +99,7 @@ PyObject* subscript(ExtensionDict* self, PyObject* key) { if (descriptor->label() != FieldDescriptor::LABEL_REPEATED && descriptor->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) { - return cmessage::InternalGetScalar(self->parent, descriptor); + return cmessage::InternalGetScalar(self->parent->message, descriptor); } PyObject* value = PyDict_GetItem(self->values, key); @@ -266,8 +256,7 @@ static PyMethodDef Methods[] = { PyTypeObject ExtensionDict_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - "google.protobuf.internal." - "cpp._message.ExtensionDict", // tp_name + FULL_MODULE_NAME ".ExtensionDict", // tp_name sizeof(ExtensionDict), // tp_basicsize 0, // tp_itemsize (destructor)extension_dict::dealloc, // tp_dealloc diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc index a2b357b2..a4843e8d 100644 --- a/python/google/protobuf/pyext/message.cc +++ b/python/google/protobuf/pyext/message.cc @@ -59,6 +59,8 @@ #include <google/protobuf/pyext/extension_dict.h> #include <google/protobuf/pyext/repeated_composite_container.h> #include <google/protobuf/pyext/repeated_scalar_container.h> +#include <google/protobuf/pyext/message_map_container.h> +#include <google/protobuf/pyext/scalar_map_container.h> #include <google/protobuf/pyext/scoped_pyobject_ptr.h> #include <google/protobuf/stubs/strutil.h> @@ -93,9 +95,9 @@ static const FieldDescriptor* GetFieldDescriptor( static const Descriptor* GetMessageDescriptor(PyTypeObject* cls); static string GetMessageName(CMessage* self); int InternalReleaseFieldByDescriptor( + CMessage* self, const FieldDescriptor* field_descriptor, - PyObject* composite_field, - Message* parent_message); + PyObject* composite_field); } // namespace cmessage // --------------------------------------------------------------------- @@ -127,10 +129,29 @@ static int VisitCompositeField(const FieldDescriptor* descriptor, Visitor visitor) { if (descriptor->label() == FieldDescriptor::LABEL_REPEATED) { if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { - RepeatedCompositeContainer* container = - reinterpret_cast<RepeatedCompositeContainer*>(child); - if (visitor.VisitRepeatedCompositeContainer(container) == -1) - return -1; + if (descriptor->is_map()) { + const Descriptor* entry_type = descriptor->message_type(); + const FieldDescriptor* value_type = + entry_type->FindFieldByName("value"); + if (value_type->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + MessageMapContainer* container = + reinterpret_cast<MessageMapContainer*>(child); + if (visitor.VisitMessageMapContainer(container) == -1) { + return -1; + } + } else { + ScalarMapContainer* container = + reinterpret_cast<ScalarMapContainer*>(child); + if (visitor.VisitScalarMapContainer(container) == -1) { + return -1; + } + } + } else { + RepeatedCompositeContainer* container = + reinterpret_cast<RepeatedCompositeContainer*>(child); + if (visitor.VisitRepeatedCompositeContainer(container) == -1) + return -1; + } } else { RepeatedScalarContainer* container = reinterpret_cast<RepeatedScalarContainer*>(child); @@ -444,7 +465,7 @@ static int MaybeReleaseOverlappingOneofField( } if (InternalReleaseFieldByDescriptor( - existing_field, child_message, message) < 0) { + cmessage, existing_field, child_message) < 0) { return -1; } return PyDict_DelItemString(cmessage->composite_fields, field_name); @@ -483,6 +504,16 @@ struct FixupMessageReference : public ChildVisitor { return 0; } + int VisitScalarMapContainer(ScalarMapContainer* container) { + container->message = message_; + return 0; + } + + int VisitMessageMapContainer(MessageMapContainer* container) { + container->message = message_; + return 0; + } + private: Message* message_; }; @@ -500,6 +531,9 @@ int AssureWritable(CMessage* self) { self->message->GetDescriptor()); self->message = prototype->New(); self->owner.reset(self->message); + // Cascade the new owner to eventual children: even if this message is + // empty, some submessages or repeated containers might exist already. + SetOwner(self, self->owner); } else { // Otherwise, we need a mutable child message. if (AssureWritable(self->parent) == -1) @@ -520,8 +554,9 @@ int AssureWritable(CMessage* self) { // When a CMessage is made writable its Message pointer is updated // to point to a new mutable Message. When that happens we need to // update any references to the old, read-only CMessage. There are - // three places such references occur: RepeatedScalarContainer, - // RepeatedCompositeContainer, and ExtensionDict. + // five places such references occur: RepeatedScalarContainer, + // RepeatedCompositeContainer, ScalarMapContainer, MessageMapContainer, + // and ExtensionDict. if (self->extensions != NULL) self->extensions->message = self->message; if (ForEachCompositeField(self, FixupMessageReference(self->message)) == -1) @@ -583,15 +618,43 @@ const FieldDescriptor* GetExtensionDescriptor(PyObject* extension) { return PyFieldDescriptor_AsDescriptor(extension); } +// If value is a string, convert it into an enum value based on the labels in +// descriptor, otherwise simply return value. Always returns a new reference. +static PyObject* GetIntegerEnumValue(const FieldDescriptor& descriptor, + PyObject* value) { + if (PyString_Check(value) || PyUnicode_Check(value)) { + const EnumDescriptor* enum_descriptor = descriptor.enum_type(); + if (enum_descriptor == NULL) { + PyErr_SetString(PyExc_TypeError, "not an enum field"); + return NULL; + } + char* enum_label; + Py_ssize_t size; + if (PyString_AsStringAndSize(value, &enum_label, &size) < 0) { + return NULL; + } + const EnumValueDescriptor* enum_value_descriptor = + enum_descriptor->FindValueByName(string(enum_label, size)); + if (enum_value_descriptor == NULL) { + PyErr_SetString(PyExc_ValueError, "unknown enum label"); + return NULL; + } + return PyInt_FromLong(enum_value_descriptor->number()); + } + Py_INCREF(value); + return value; +} + // If cmessage_list is not NULL, this function releases values into the // container CMessages instead of just removing. Repeated composite container // needs to do this to make sure CMessages stay alive if they're still // referenced after deletion. Repeated scalar container doesn't need to worry. int InternalDeleteRepeatedField( - Message* message, + CMessage* self, const FieldDescriptor* field_descriptor, PyObject* slice, PyObject* cmessage_list) { + Message* message = self->message; Py_ssize_t length, from, to, step, slice_length; const Reflection* reflection = message->GetReflection(); int min, max; @@ -665,7 +728,7 @@ int InternalDeleteRepeatedField( CMessage* last_cmessage = reinterpret_cast<CMessage*>( PyList_GET_ITEM(cmessage_list, PyList_GET_SIZE(cmessage_list) - 1)); repeated_composite_container::ReleaseLastTo( - field_descriptor, message, last_cmessage); + self, field_descriptor, last_cmessage); if (PySequence_DelItem(cmessage_list, -1) < 0) { return -1; } @@ -696,16 +759,90 @@ int InitAttributes(CMessage* self, PyObject* kwargs) { PyString_AsString(name)); return -1; } - if (descriptor->label() == FieldDescriptor::LABEL_REPEATED) { + if (descriptor->is_map()) { + ScopedPyObjectPtr map(GetAttr(self, name)); + const FieldDescriptor* value_descriptor = + descriptor->message_type()->FindFieldByName("value"); + if (value_descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + Py_ssize_t map_pos = 0; + PyObject* map_key; + PyObject* map_value; + while (PyDict_Next(value, &map_pos, &map_key, &map_value)) { + ScopedPyObjectPtr function_return; + function_return.reset(PyObject_GetItem(map.get(), map_key)); + if (function_return.get() == NULL) { + return -1; + } + ScopedPyObjectPtr ok(PyObject_CallMethod( + function_return.get(), "MergeFrom", "O", map_value)); + if (ok.get() == NULL) { + return -1; + } + } + } else { + ScopedPyObjectPtr function_return; + function_return.reset( + PyObject_CallMethod(map.get(), "update", "O", value)); + if (function_return.get() == NULL) { + return -1; + } + } + } else if (descriptor->label() == FieldDescriptor::LABEL_REPEATED) { ScopedPyObjectPtr container(GetAttr(self, name)); if (container == NULL) { return -1; } if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { - if (repeated_composite_container::Extend( - reinterpret_cast<RepeatedCompositeContainer*>(container.get()), - value) - == NULL) { + RepeatedCompositeContainer* rc_container = + reinterpret_cast<RepeatedCompositeContainer*>(container.get()); + ScopedPyObjectPtr iter(PyObject_GetIter(value)); + if (iter == NULL) { + PyErr_SetString(PyExc_TypeError, "Value must be iterable"); + return -1; + } + ScopedPyObjectPtr next; + while ((next.reset(PyIter_Next(iter))) != NULL) { + PyObject* kwargs = (PyDict_Check(next) ? next.get() : NULL); + ScopedPyObjectPtr new_msg( + repeated_composite_container::Add(rc_container, NULL, kwargs)); + if (new_msg == NULL) { + return -1; + } + if (kwargs == NULL) { + // next was not a dict, it's a message we need to merge + ScopedPyObjectPtr merged( + MergeFrom(reinterpret_cast<CMessage*>(new_msg.get()), next)); + if (merged == NULL) { + return -1; + } + } + } + if (PyErr_Occurred()) { + // Check to see how PyIter_Next() exited. + return -1; + } + } else if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_ENUM) { + RepeatedScalarContainer* rs_container = + reinterpret_cast<RepeatedScalarContainer*>(container.get()); + ScopedPyObjectPtr iter(PyObject_GetIter(value)); + if (iter == NULL) { + PyErr_SetString(PyExc_TypeError, "Value must be iterable"); + return -1; + } + ScopedPyObjectPtr next; + while ((next.reset(PyIter_Next(iter))) != NULL) { + ScopedPyObjectPtr enum_value(GetIntegerEnumValue(*descriptor, next)); + if (enum_value == NULL) { + return -1; + } + ScopedPyObjectPtr new_msg( + repeated_scalar_container::Append(rs_container, enum_value)); + if (new_msg == NULL) { + return -1; + } + } + if (PyErr_Occurred()) { + // Check to see how PyIter_Next() exited. return -1; } } else { @@ -721,12 +858,26 @@ int InitAttributes(CMessage* self, PyObject* kwargs) { if (message == NULL) { return -1; } - if (MergeFrom(reinterpret_cast<CMessage*>(message.get()), - value) == NULL) { - return -1; + CMessage* cmessage = reinterpret_cast<CMessage*>(message.get()); + if (PyDict_Check(value)) { + if (InitAttributes(cmessage, value) < 0) { + return -1; + } + } else { + ScopedPyObjectPtr merged(MergeFrom(cmessage, value)); + if (merged == NULL) { + return -1; + } } } else { - if (SetAttr(self, name, value) < 0) { + ScopedPyObjectPtr new_val; + if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_ENUM) { + new_val.reset(GetIntegerEnumValue(*descriptor, value)); + if (new_val == NULL) { + return -1; + } + } + if (SetAttr(self, name, (new_val == NULL) ? value : new_val) < 0) { return -1; } } @@ -789,7 +940,6 @@ static PyObject* New(PyTypeObject* type, } self->message = default_message->New(); self->owner.reset(self->message); - return reinterpret_cast<PyObject*>(self); } @@ -830,6 +980,16 @@ struct ClearWeakReferences : public ChildVisitor { return 0; } + int VisitScalarMapContainer(ScalarMapContainer* container) { + container->parent = NULL; + return 0; + } + + int VisitMessageMapContainer(MessageMapContainer* container) { + container->parent = NULL; + return 0; + } + int VisitCMessage(CMessage* cmessage, const FieldDescriptor* field_descriptor) { cmessage->parent = NULL; @@ -1064,6 +1224,16 @@ struct SetOwnerVisitor : public ChildVisitor { return 0; } + int VisitScalarMapContainer(ScalarMapContainer* container) { + scalar_map_container::SetOwner(container, new_owner_); + return 0; + } + + int VisitMessageMapContainer(MessageMapContainer* container) { + message_map_container::SetOwner(container, new_owner_); + return 0; + } + int VisitCMessage(CMessage* cmessage, const FieldDescriptor* field_descriptor) { return SetOwner(cmessage, new_owner_); @@ -1084,11 +1254,11 @@ int SetOwner(CMessage* self, const shared_ptr<Message>& new_owner) { // Releases the message specified by 'field' and returns the // pointer. If the field does not exist a new message is created using // 'descriptor'. The caller takes ownership of the returned pointer. -Message* ReleaseMessage(Message* message, +Message* ReleaseMessage(CMessage* self, const Descriptor* descriptor, const FieldDescriptor* field_descriptor) { - Message* released_message = message->GetReflection()->ReleaseMessage( - message, field_descriptor, message_factory); + Message* released_message = self->message->GetReflection()->ReleaseMessage( + self->message, field_descriptor, message_factory); // ReleaseMessage will return NULL which differs from // child_cmessage->message, if the field does not exist. In this case, // the latter points to the default instance via a const_cast<>, so we @@ -1102,12 +1272,12 @@ Message* ReleaseMessage(Message* message, return released_message; } -int ReleaseSubMessage(Message* message, +int ReleaseSubMessage(CMessage* self, const FieldDescriptor* field_descriptor, CMessage* child_cmessage) { // Release the Message shared_ptr<Message> released_message(ReleaseMessage( - message, child_cmessage->message->GetDescriptor(), field_descriptor)); + self, child_cmessage->message->GetDescriptor(), field_descriptor)); child_cmessage->message = released_message.get(); child_cmessage->owner.swap(released_message); child_cmessage->parent = NULL; @@ -1119,8 +1289,8 @@ int ReleaseSubMessage(Message* message, struct ReleaseChild : public ChildVisitor { // message must outlive this object. - explicit ReleaseChild(Message* parent_message) : - parent_message_(parent_message) {} + explicit ReleaseChild(CMessage* parent) : + parent_(parent) {} int VisitRepeatedCompositeContainer(RepeatedCompositeContainer* container) { return repeated_composite_container::Release( @@ -1132,23 +1302,33 @@ struct ReleaseChild : public ChildVisitor { reinterpret_cast<RepeatedScalarContainer*>(container)); } + int VisitScalarMapContainer(ScalarMapContainer* container) { + return scalar_map_container::Release( + reinterpret_cast<ScalarMapContainer*>(container)); + } + + int VisitMessageMapContainer(MessageMapContainer* container) { + return message_map_container::Release( + reinterpret_cast<MessageMapContainer*>(container)); + } + int VisitCMessage(CMessage* cmessage, const FieldDescriptor* field_descriptor) { - return ReleaseSubMessage(parent_message_, field_descriptor, + return ReleaseSubMessage(parent_, field_descriptor, reinterpret_cast<CMessage*>(cmessage)); } - Message* parent_message_; + CMessage* parent_; }; int InternalReleaseFieldByDescriptor( + CMessage* self, const FieldDescriptor* field_descriptor, - PyObject* composite_field, - Message* parent_message) { + PyObject* composite_field) { return VisitCompositeField( field_descriptor, composite_field, - ReleaseChild(parent_message)); + ReleaseChild(self)); } PyObject* ClearFieldByDescriptor( @@ -1200,8 +1380,8 @@ PyObject* ClearField(CMessage* self, PyObject* arg) { // Only release the field if there's a possibility that there are // references to it. if (composite_field != NULL) { - if (InternalReleaseFieldByDescriptor(field_descriptor, - composite_field, message) < 0) { + if (InternalReleaseFieldByDescriptor(self, field_descriptor, + composite_field) < 0) { return NULL; } PyDict_DelItem(self->composite_fields, arg); @@ -1219,7 +1399,7 @@ PyObject* ClearField(CMessage* self, PyObject* arg) { PyObject* Clear(CMessage* self) { AssureWritable(self); - if (ForEachCompositeField(self, ReleaseChild(self->message)) == -1) + if (ForEachCompositeField(self, ReleaseChild(self)) == -1) return NULL; // The old ExtensionDict still aliases this CMessage, but all its @@ -1582,7 +1762,8 @@ static PyObject* ListFields(CMessage* self) { } if (fields[i]->is_extension()) { - ScopedPyObjectPtr extension_field(PyFieldDescriptor_New(fields[i])); + ScopedPyObjectPtr extension_field( + PyFieldDescriptor_FromDescriptor(fields[i])); if (extension_field == NULL) { return NULL; } @@ -1616,7 +1797,8 @@ static PyObject* ListFields(CMessage* self) { PyErr_SetString(PyExc_ValueError, "bad string"); return NULL; } - ScopedPyObjectPtr field_descriptor(PyFieldDescriptor_New(fields[i])); + ScopedPyObjectPtr field_descriptor( + PyFieldDescriptor_FromDescriptor(fields[i])); if (field_descriptor == NULL) { return NULL; } @@ -1683,10 +1865,8 @@ static PyObject* RichCompare(CMessage* self, PyObject* other, int opid) { } } -PyObject* InternalGetScalar( - CMessage* self, - const FieldDescriptor* field_descriptor) { - Message* message = self->message; +PyObject* InternalGetScalar(const Message* message, + const FieldDescriptor* field_descriptor) { const Reflection* reflection = message->GetReflection(); if (!CheckFieldBelongsToMessage(field_descriptor, message)) { @@ -1739,12 +1919,12 @@ PyObject* InternalGetScalar( if (!message->GetReflection()->SupportsUnknownEnumValues() && !message->GetReflection()->HasField(*message, field_descriptor)) { // Look for the value in the unknown fields. - UnknownFieldSet* unknown_field_set = - message->GetReflection()->MutableUnknownFields(message); - for (int i = 0; i < unknown_field_set->field_count(); ++i) { - if (unknown_field_set->field(i).number() == + const UnknownFieldSet& unknown_field_set = + message->GetReflection()->GetUnknownFields(*message); + for (int i = 0; i < unknown_field_set.field_count(); ++i) { + if (unknown_field_set.field(i).number() == field_descriptor->number()) { - result = PyInt_FromLong(unknown_field_set->field(i).varint()); + result = PyInt_FromLong(unknown_field_set.field(i).varint()); break; } } @@ -1793,21 +1973,16 @@ PyObject* InternalGetSubMessage( return reinterpret_cast<PyObject*>(cmsg); } -int InternalSetScalar( - CMessage* self, +int InternalSetNonOneofScalar( + Message* message, const FieldDescriptor* field_descriptor, PyObject* arg) { - Message* message = self->message; const Reflection* reflection = message->GetReflection(); if (!CheckFieldBelongsToMessage(field_descriptor, message)) { return -1; } - if (MaybeReleaseOverlappingOneofField(self, field_descriptor) < 0) { - return -1; - } - switch (field_descriptor->cpp_type()) { case FieldDescriptor::CPPTYPE_INT32: { GOOGLE_CHECK_GET_INT32(arg, value, -1); @@ -1878,6 +2053,21 @@ int InternalSetScalar( return 0; } +int InternalSetScalar( + CMessage* self, + const FieldDescriptor* field_descriptor, + PyObject* arg) { + if (!CheckFieldBelongsToMessage(field_descriptor, self->message)) { + return -1; + } + + if (MaybeReleaseOverlappingOneofField(self, field_descriptor) < 0) { + return -1; + } + + return InternalSetNonOneofScalar(self->message, field_descriptor, arg); +} + PyObject* FromString(PyTypeObject* cls, PyObject* serialized) { PyObject* py_cmsg = PyObject_CallObject( reinterpret_cast<PyObject*>(cls), NULL); @@ -1955,7 +2145,8 @@ static PyObject* AddDescriptors(PyObject* cls, PyObject* descriptor) { // which was built previously. for (int i = 0; i < message_descriptor->enum_type_count(); ++i) { const EnumDescriptor* enum_descriptor = message_descriptor->enum_type(i); - ScopedPyObjectPtr enum_type(PyEnumDescriptor_New(enum_descriptor)); + ScopedPyObjectPtr enum_type( + PyEnumDescriptor_FromDescriptor(enum_descriptor)); if (enum_type == NULL) { return NULL; } @@ -1993,7 +2184,7 @@ static PyObject* AddDescriptors(PyObject* cls, PyObject* descriptor) { // which was defined previously. for (int i = 0; i < message_descriptor->extension_count(); ++i) { const google::protobuf::FieldDescriptor* field = message_descriptor->extension(i); - ScopedPyObjectPtr extension_field(PyFieldDescriptor_New(field)); + ScopedPyObjectPtr extension_field(PyFieldDescriptor_FromDescriptor(field)); if (extension_field == NULL) { return NULL; } @@ -2097,26 +2288,6 @@ PyObject* SetState(CMessage* self, PyObject* state) { } // CMessage static methods: -PyObject* _GetMessageDescriptor(PyObject* unused, PyObject* arg) { - return cdescriptor_pool::FindMessageByName(GetDescriptorPool(), arg); -} - -PyObject* _GetFieldDescriptor(PyObject* unused, PyObject* arg) { - return cdescriptor_pool::FindFieldByName(GetDescriptorPool(), arg); -} - -PyObject* _GetExtensionDescriptor(PyObject* unused, PyObject* arg) { - return cdescriptor_pool::FindExtensionByName(GetDescriptorPool(), arg); -} - -PyObject* _GetEnumDescriptor(PyObject* unused, PyObject* arg) { - return cdescriptor_pool::FindEnumTypeByName(GetDescriptorPool(), arg); -} - -PyObject* _GetOneofDescriptor(PyObject* unused, PyObject* arg) { - return cdescriptor_pool::FindOneofByName(GetDescriptorPool(), arg); -} - PyObject* _CheckCalledFromGeneratedFile(PyObject* unused, PyObject* unused_arg) { if (!_CalledFromGeneratedFile(1)) { @@ -2188,21 +2359,6 @@ static PyMethodDef Methods[] = { "or None if no field is set." }, // Static Methods. - { "_BuildFile", (PyCFunction)Python_BuildFile, METH_O | METH_STATIC, - "Registers a new protocol buffer file in the global C++ descriptor pool." }, - { "_GetMessageDescriptor", (PyCFunction)_GetMessageDescriptor, - METH_O | METH_STATIC, "Finds a message descriptor in the message pool." }, - { "_GetFieldDescriptor", (PyCFunction)_GetFieldDescriptor, - METH_O | METH_STATIC, "Finds a field descriptor in the message pool." }, - { "_GetExtensionDescriptor", (PyCFunction)_GetExtensionDescriptor, - METH_O | METH_STATIC, - "Finds a extension descriptor in the message pool." }, - { "_GetEnumDescriptor", (PyCFunction)_GetEnumDescriptor, - METH_O | METH_STATIC, - "Finds an enum descriptor in the message pool." }, - { "_GetOneofDescriptor", (PyCFunction)_GetOneofDescriptor, - METH_O | METH_STATIC, - "Finds an oneof descriptor in the message pool." }, { "_CheckCalledFromGeneratedFile", (PyCFunction)_CheckCalledFromGeneratedFile, METH_NOARGS | METH_STATIC, "Raises TypeError if the caller is not in a _pb2.py file."}, @@ -2234,6 +2390,31 @@ PyObject* GetAttr(CMessage* self, PyObject* name) { reinterpret_cast<PyObject*>(self), name); } + if (field_descriptor->is_map()) { + PyObject* py_container = NULL; + const Descriptor* entry_type = field_descriptor->message_type(); + const FieldDescriptor* value_type = entry_type->FindFieldByName("value"); + if (value_type->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + PyObject* value_class = cdescriptor_pool::GetMessageClass( + GetDescriptorPool(), value_type->message_type()); + if (value_class == NULL) { + return NULL; + } + py_container = message_map_container::NewContainer(self, field_descriptor, + value_class); + } else { + py_container = scalar_map_container::NewContainer(self, field_descriptor); + } + if (py_container == NULL) { + return NULL; + } + if (!SetCompositeField(self, name, py_container)) { + Py_DECREF(py_container); + return NULL; + } + return py_container; + } + if (field_descriptor->label() == FieldDescriptor::LABEL_REPEATED) { PyObject* py_container = NULL; if (field_descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { @@ -2267,7 +2448,7 @@ PyObject* GetAttr(CMessage* self, PyObject* name) { return sub_message; } - return InternalGetScalar(self, field_descriptor); + return InternalGetScalar(self->message, field_descriptor); } int SetAttr(CMessage* self, PyObject* name, PyObject* value) { @@ -2304,9 +2485,7 @@ int SetAttr(CMessage* self, PyObject* name, PyObject* value) { PyTypeObject CMessage_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - // Keep the fully qualified _message symbol in a line for opensource. - "google.protobuf.pyext._message." - "CMessage", // tp_name + FULL_MODULE_NAME ".CMessage", // tp_name sizeof(CMessage), // tp_basicsize 0, // tp_itemsize (destructor)cmessage::Dealloc, // tp_dealloc @@ -2401,7 +2580,7 @@ void InitGlobals() { k_extensions_by_name = PyString_FromString("_extensions_by_name"); k_extensions_by_number = PyString_FromString("_extensions_by_number"); - message_factory = new DynamicMessageFactory(GetDescriptorPool()->pool); + message_factory = new DynamicMessageFactory(); message_factory->SetDelegateToGeneratedFactory(true); } @@ -2469,6 +2648,61 @@ bool InitProto2MessageModule(PyObject *m) { reinterpret_cast<PyObject*>( &RepeatedCompositeContainer_Type)); + // ScalarMapContainer_Type derives from our MutableMapping type. + PyObject* containers = + PyImport_ImportModule("google.protobuf.internal.containers"); + if (containers == NULL) { + return false; + } + + PyObject* mutable_mapping = + PyObject_GetAttrString(containers, "MutableMapping"); + Py_DECREF(containers); + + if (mutable_mapping == NULL) { + return false; + } + + if (!PyObject_TypeCheck(mutable_mapping, &PyType_Type)) { + Py_DECREF(mutable_mapping); + return false; + } + + ScalarMapContainer_Type.tp_base = + reinterpret_cast<PyTypeObject*>(mutable_mapping); + + if (PyType_Ready(&ScalarMapContainer_Type) < 0) { + return false; + } + + PyModule_AddObject(m, "ScalarMapContainer", + reinterpret_cast<PyObject*>(&ScalarMapContainer_Type)); + + if (PyType_Ready(&ScalarMapIterator_Type) < 0) { + return false; + } + + PyModule_AddObject(m, "ScalarMapIterator", + reinterpret_cast<PyObject*>(&ScalarMapIterator_Type)); + + Py_INCREF(mutable_mapping); + MessageMapContainer_Type.tp_base = + reinterpret_cast<PyTypeObject*>(mutable_mapping); + + if (PyType_Ready(&MessageMapContainer_Type) < 0) { + return false; + } + + PyModule_AddObject(m, "MessageMapContainer", + reinterpret_cast<PyObject*>(&MessageMapContainer_Type)); + + if (PyType_Ready(&MessageMapIterator_Type) < 0) { + return false; + } + + PyModule_AddObject(m, "MessageMapIterator", + reinterpret_cast<PyObject*>(&MessageMapIterator_Type)); + ExtensionDict_Type.tp_hash = PyObject_HashNotImplemented; if (PyType_Ready(&ExtensionDict_Type) < 0) { return false; @@ -2478,6 +2712,12 @@ bool InitProto2MessageModule(PyObject *m) { m, "ExtensionDict", reinterpret_cast<PyObject*>(&ExtensionDict_Type)); + // Expose the DescriptorPool used to hold all descriptors added from generated + // pb2.py files. + Py_INCREF(GetDescriptorPool()); // PyModule_AddObject steals a reference. + PyModule_AddObject( + m, "default_pool", reinterpret_cast<PyObject*>(GetDescriptorPool())); + // This implementation provides full Descriptor types, we advertise it so that // descriptor.py can use them in replacement of the Python classes. PyModule_AddIntConstant(m, "_USE_C_DESCRIPTORS", 1); diff --git a/python/google/protobuf/pyext/message.h b/python/google/protobuf/pyext/message.h index 2f2da795..7360b207 100644 --- a/python/google/protobuf/pyext/message.h +++ b/python/google/protobuf/pyext/message.h @@ -120,7 +120,7 @@ CMessage* NewEmptyMessage(PyObject* type, const Descriptor* descriptor); // A new message will be created if this is a read-only default instance. // // Corresponds to reflection api method ReleaseMessage. -int ReleaseSubMessage(Message* message, +int ReleaseSubMessage(CMessage* self, const FieldDescriptor* field_descriptor, CMessage* child_cmessage); @@ -144,7 +144,7 @@ PyObject* InternalGetSubMessage( // by slice will be removed from cmessage_list by this function. // // Corresponds to reflection api method RemoveLast. -int InternalDeleteRepeatedField(Message* message, +int InternalDeleteRepeatedField(CMessage* self, const FieldDescriptor* field_descriptor, PyObject* slice, PyObject* cmessage_list); @@ -153,10 +153,15 @@ int InternalSetScalar(CMessage* self, const FieldDescriptor* field_descriptor, PyObject* value); +// Sets the specified scalar value to the message. Requires it is not a Oneof. +int InternalSetNonOneofScalar(Message* message, + const FieldDescriptor* field_descriptor, + PyObject* arg); + // Retrieves the specified scalar value from the message. // // Returns a new python reference. -PyObject* InternalGetScalar(CMessage* self, +PyObject* InternalGetScalar(const Message* message, const FieldDescriptor* field_descriptor); // Clears the message, removing all contained data. Extension dictionary and @@ -279,7 +284,7 @@ extern PyObject* kint64min_py; extern PyObject* kint64max_py; extern PyObject* kuint64max_py; -#define C(str) const_cast<char*>(str) +#define FULL_MODULE_NAME "google.protobuf.pyext._message" void FormatTypeError(PyObject* arg, char* expected_types); template<class T> diff --git a/python/google/protobuf/pyext/message_map_container.cc b/python/google/protobuf/pyext/message_map_container.cc new file mode 100644 index 00000000..ab8d8fb9 --- /dev/null +++ b/python/google/protobuf/pyext/message_map_container.cc @@ -0,0 +1,540 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: haberman@google.com (Josh Haberman) + +#include <google/protobuf/pyext/message_map_container.h> + +#include <google/protobuf/stubs/common.h> +#include <google/protobuf/message.h> +#include <google/protobuf/pyext/message.h> +#include <google/protobuf/pyext/scoped_pyobject_ptr.h> + +namespace google { +namespace protobuf { +namespace python { + +struct MessageMapIterator { + PyObject_HEAD; + + // This dict contains the full contents of what we want to iterate over. + // There's no way to avoid building this, because the list representation + // (which is canonical) can contain duplicate keys. So at the very least we + // need a set that lets us skip duplicate keys. And at the point that we're + // doing that, we might as well just build the actual dict we're iterating + // over and use dict's built-in iterator. + PyObject* dict; + + // An iterator on dict. + PyObject* iter; + + // A pointer back to the container, so we can notice changes to the version. + MessageMapContainer* container; + + // The version of the map when we took the iterator to it. + // + // We store this so that if the map is modified during iteration we can throw + // an error. + uint64 version; +}; + +static MessageMapIterator* GetIter(PyObject* obj) { + return reinterpret_cast<MessageMapIterator*>(obj); +} + +namespace message_map_container { + +static MessageMapContainer* GetMap(PyObject* obj) { + return reinterpret_cast<MessageMapContainer*>(obj); +} + +// The private constructor of MessageMapContainer objects. +PyObject* NewContainer(CMessage* parent, + const google::protobuf::FieldDescriptor* parent_field_descriptor, + PyObject* concrete_class) { + if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) { + return NULL; + } + + PyObject* obj = PyType_GenericAlloc(&MessageMapContainer_Type, 0); + if (obj == NULL) { + return PyErr_Format(PyExc_RuntimeError, + "Could not allocate new container."); + } + + MessageMapContainer* self = GetMap(obj); + + self->message = parent->message; + self->parent = parent; + self->parent_field_descriptor = parent_field_descriptor; + self->owner = parent->owner; + self->version = 0; + + self->key_field_descriptor = + parent_field_descriptor->message_type()->FindFieldByName("key"); + self->value_field_descriptor = + parent_field_descriptor->message_type()->FindFieldByName("value"); + + self->message_dict = PyDict_New(); + if (self->message_dict == NULL) { + return PyErr_Format(PyExc_RuntimeError, + "Could not allocate message dict."); + } + + Py_INCREF(concrete_class); + self->subclass_init = concrete_class; + + if (self->key_field_descriptor == NULL || + self->value_field_descriptor == NULL) { + Py_DECREF(obj); + return PyErr_Format(PyExc_KeyError, + "Map entry descriptor did not have key/value fields"); + } + + return obj; +} + +// Initializes the underlying Message object of "to" so it becomes a new parent +// repeated scalar, and copies all the values from "from" to it. A child scalar +// container can be released by passing it as both from and to (e.g. making it +// the recipient of the new parent message and copying the values from itself). +static int InitializeAndCopyToParentContainer( + MessageMapContainer* from, + MessageMapContainer* to) { + // For now we require from == to, re-evaluate if we want to support deep copy + // as in repeated_composite_container.cc. + GOOGLE_DCHECK(from == to); + Message* old_message = from->message; + Message* new_message = old_message->New(); + to->parent = NULL; + to->parent_field_descriptor = from->parent_field_descriptor; + to->message = new_message; + to->owner.reset(new_message); + + vector<const FieldDescriptor*> fields; + fields.push_back(from->parent_field_descriptor); + old_message->GetReflection()->SwapFields(old_message, new_message, fields); + return 0; +} + +static PyObject* GetCMessage(MessageMapContainer* self, Message* entry) { + // Get or create the CMessage object corresponding to this message. + Message* message = entry->GetReflection()->MutableMessage( + entry, self->value_field_descriptor); + ScopedPyObjectPtr key(PyLong_FromVoidPtr(message)); + PyObject* ret = PyDict_GetItem(self->message_dict, key); + + if (ret == NULL) { + CMessage* cmsg = cmessage::NewEmptyMessage(self->subclass_init, + message->GetDescriptor()); + ret = reinterpret_cast<PyObject*>(cmsg); + + if (cmsg == NULL) { + return NULL; + } + cmsg->owner = self->owner; + cmsg->message = message; + cmsg->parent = self->parent; + + if (PyDict_SetItem(self->message_dict, key, ret) < 0) { + Py_DECREF(ret); + return NULL; + } + } else { + Py_INCREF(ret); + } + + return ret; +} + +int Release(MessageMapContainer* self) { + InitializeAndCopyToParentContainer(self, self); + return 0; +} + +void SetOwner(MessageMapContainer* self, + const shared_ptr<Message>& new_owner) { + self->owner = new_owner; +} + +Py_ssize_t Length(PyObject* _self) { + MessageMapContainer* self = GetMap(_self); + google::protobuf::Message* message = self->message; + return message->GetReflection()->FieldSize(*message, + self->parent_field_descriptor); +} + +int MapKeyMatches(MessageMapContainer* self, const Message* entry, + PyObject* key) { + // TODO(haberman): do we need more strict type checking? + ScopedPyObjectPtr entry_key( + cmessage::InternalGetScalar(entry, self->key_field_descriptor)); + int ret = PyObject_RichCompareBool(key, entry_key, Py_EQ); + return ret; +} + +int SetItem(PyObject *_self, PyObject *key, PyObject *v) { + if (v) { + PyErr_Format(PyExc_ValueError, + "Direct assignment of submessage not allowed"); + return -1; + } + + // Now we know that this is a delete, not a set. + + MessageMapContainer* self = GetMap(_self); + cmessage::AssureWritable(self->parent); + + Message* message = self->message; + const Reflection* reflection = message->GetReflection(); + size_t size = + reflection->FieldSize(*message, self->parent_field_descriptor); + + // Right now the Reflection API doesn't support map lookup, so we implement it + // via linear search. We need to search from the end because the underlying + // representation can have duplicates if a user calls MergeFrom(); the last + // one needs to win. + // + // TODO(haberman): add lookup API to Reflection API. + bool found = false; + for (int i = size - 1; i >= 0; i--) { + Message* entry = reflection->MutableRepeatedMessage( + message, self->parent_field_descriptor, i); + int matches = MapKeyMatches(self, entry, key); + if (matches < 0) return -1; + if (matches) { + found = true; + if (i != size - 1) { + reflection->SwapElements(message, self->parent_field_descriptor, i, + size - 1); + } + reflection->RemoveLast(message, self->parent_field_descriptor); + + // Can't exit now, the repeated field representation of maps allows + // duplicate keys, and we have to be sure to remove all of them. + } + } + + if (!found) { + PyErr_Format(PyExc_KeyError, "Key not present in map"); + return -1; + } + + self->version++; + + return 0; +} + +PyObject* GetIterator(PyObject *_self) { + MessageMapContainer* self = GetMap(_self); + + ScopedPyObjectPtr obj(PyType_GenericAlloc(&MessageMapIterator_Type, 0)); + if (obj == NULL) { + return PyErr_Format(PyExc_KeyError, "Could not allocate iterator"); + } + + MessageMapIterator* iter = GetIter(obj); + + Py_INCREF(self); + iter->container = self; + iter->version = self->version; + iter->dict = PyDict_New(); + if (iter->dict == NULL) { + return PyErr_Format(PyExc_RuntimeError, + "Could not allocate dict for iterator."); + } + + // Build the entire map into a dict right now. Start from the beginning so + // that later entries win in the case of duplicates. + Message* message = self->message; + const Reflection* reflection = message->GetReflection(); + + // Right now the Reflection API doesn't support map lookup, so we implement it + // via linear search. We need to search from the end because the underlying + // representation can have duplicates if a user calls MergeFrom(); the last + // one needs to win. + // + // TODO(haberman): add lookup API to Reflection API. + size_t size = + reflection->FieldSize(*message, self->parent_field_descriptor); + for (int i = size - 1; i >= 0; i--) { + Message* entry = reflection->MutableRepeatedMessage( + message, self->parent_field_descriptor, i); + ScopedPyObjectPtr key( + cmessage::InternalGetScalar(entry, self->key_field_descriptor)); + if (PyDict_SetItem(iter->dict, key.get(), GetCMessage(self, entry)) < 0) { + return PyErr_Format(PyExc_RuntimeError, + "SetItem failed in iterator construction."); + } + } + + iter->iter = PyObject_GetIter(iter->dict); + + return obj.release(); +} + +PyObject* GetItem(PyObject* _self, PyObject* key) { + MessageMapContainer* self = GetMap(_self); + cmessage::AssureWritable(self->parent); + Message* message = self->message; + const Reflection* reflection = message->GetReflection(); + + // Right now the Reflection API doesn't support map lookup, so we implement it + // via linear search. We need to search from the end because the underlying + // representation can have duplicates if a user calls MergeFrom(); the last + // one needs to win. + // + // TODO(haberman): add lookup API to Reflection API. + size_t size = + reflection->FieldSize(*message, self->parent_field_descriptor); + for (int i = size - 1; i >= 0; i--) { + Message* entry = reflection->MutableRepeatedMessage( + message, self->parent_field_descriptor, i); + int matches = MapKeyMatches(self, entry, key); + if (matches < 0) return NULL; + if (matches) { + return GetCMessage(self, entry); + } + } + + // Key is not already present; insert a new entry. + Message* entry = + reflection->AddMessage(message, self->parent_field_descriptor); + + self->version++; + + if (cmessage::InternalSetNonOneofScalar(entry, self->key_field_descriptor, + key) < 0) { + reflection->RemoveLast(message, self->parent_field_descriptor); + return NULL; + } + + return GetCMessage(self, entry); +} + +PyObject* Contains(PyObject* _self, PyObject* key) { + MessageMapContainer* self = GetMap(_self); + Message* message = self->message; + const Reflection* reflection = message->GetReflection(); + + // Right now the Reflection API doesn't support map lookup, so we implement it + // via linear search. + // + // TODO(haberman): add lookup API to Reflection API. + size_t size = + reflection->FieldSize(*message, self->parent_field_descriptor); + for (int i = 0; i < size; i++) { + Message* entry = reflection->MutableRepeatedMessage( + message, self->parent_field_descriptor, i); + int matches = MapKeyMatches(self, entry, key); + if (matches < 0) return NULL; + if (matches) { + Py_RETURN_TRUE; + } + } + + Py_RETURN_FALSE; +} + +PyObject* Clear(PyObject* _self) { + MessageMapContainer* self = GetMap(_self); + cmessage::AssureWritable(self->parent); + Message* message = self->message; + const Reflection* reflection = message->GetReflection(); + + self->version++; + reflection->ClearField(message, self->parent_field_descriptor); + + Py_RETURN_NONE; +} + +PyObject* Get(PyObject* self, PyObject* args) { + PyObject* key; + PyObject* default_value = NULL; + if (PyArg_ParseTuple(args, "O|O", &key, &default_value) < 0) { + return NULL; + } + + ScopedPyObjectPtr is_present(Contains(self, key)); + if (is_present.get() == NULL) { + return NULL; + } + + if (PyObject_IsTrue(is_present.get())) { + return GetItem(self, key); + } else { + if (default_value != NULL) { + Py_INCREF(default_value); + return default_value; + } else { + Py_RETURN_NONE; + } + } +} + +static PyMappingMethods MpMethods = { + Length, // mp_length + GetItem, // mp_subscript + SetItem, // mp_ass_subscript +}; + +static void Dealloc(PyObject* _self) { + MessageMapContainer* self = GetMap(_self); + self->owner.reset(); + Py_DECREF(self->message_dict); + Py_TYPE(_self)->tp_free(_self); +} + +static PyMethodDef Methods[] = { + { "__contains__", (PyCFunction)Contains, METH_O, + "Tests whether the map contains this element."}, + { "clear", (PyCFunction)Clear, METH_NOARGS, + "Removes all elements from the map."}, + { "get", Get, METH_VARARGS, + "Gets the value for the given key if present, or otherwise a default" }, + { "get_or_create", GetItem, METH_O, + "Alias for getitem, useful to make explicit that the map is mutated." }, + /* + { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS, + "Makes a deep copy of the class." }, + { "__reduce__", (PyCFunction)Reduce, METH_NOARGS, + "Outputs picklable representation of the repeated field." }, + */ + {NULL, NULL}, +}; + +} // namespace message_map_container + +namespace message_map_iterator { + +static void Dealloc(PyObject* _self) { + MessageMapIterator* self = GetIter(_self); + Py_DECREF(self->dict); + Py_DECREF(self->iter); + Py_DECREF(self->container); + Py_TYPE(_self)->tp_free(_self); +} + +PyObject* IterNext(PyObject* _self) { + MessageMapIterator* self = GetIter(_self); + + // This won't catch mutations to the map performed by MergeFrom(); no easy way + // to address that. + if (self->version != self->container->version) { + return PyErr_Format(PyExc_RuntimeError, + "Map modified during iteration."); + } + + return PyIter_Next(self->iter); +} + +} // namespace message_map_iterator + +PyTypeObject MessageMapContainer_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + FULL_MODULE_NAME ".MessageMapContainer", // tp_name + sizeof(MessageMapContainer), // tp_basicsize + 0, // tp_itemsize + message_map_container::Dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + &message_map_container::MpMethods, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "A map container for message", // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + message_map_container::GetIterator, // tp_iter + 0, // tp_iternext + message_map_container::Methods, // tp_methods + 0, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init +}; + +PyTypeObject MessageMapIterator_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + FULL_MODULE_NAME ".MessageMapIterator", // tp_name + sizeof(MessageMapIterator), // tp_basicsize + 0, // tp_itemsize + message_map_iterator::Dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "A scalar map iterator", // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + PyObject_SelfIter, // tp_iter + message_map_iterator::IterNext, // tp_iternext + 0, // tp_methods + 0, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init +}; + +} // namespace python +} // namespace protobuf +} // namespace google diff --git a/python/google/protobuf/pyext/message_map_container.h b/python/google/protobuf/pyext/message_map_container.h new file mode 100644 index 00000000..4ca0aecc --- /dev/null +++ b/python/google/protobuf/pyext/message_map_container.h @@ -0,0 +1,117 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_MAP_CONTAINER_H__ +#define GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_MAP_CONTAINER_H__ + +#include <Python.h> + +#include <memory> +#ifndef _SHARED_PTR_H +#include <google/protobuf/stubs/shared_ptr.h> +#endif + +#include <google/protobuf/descriptor.h> + +namespace google { +namespace protobuf { + +class Message; + +using internal::shared_ptr; + +namespace python { + +struct CMessage; + +struct MessageMapContainer { + PyObject_HEAD; + + // This is the top-level C++ Message object that owns the whole + // proto tree. Every Python MessageMapContainer holds a + // reference to it in order to keep it alive as long as there's a + // Python object that references any part of the tree. + shared_ptr<Message> owner; + + // Pointer to the C++ Message that contains this container. The + // MessageMapContainer does not own this pointer. + Message* message; + + // Weak reference to a parent CMessage object (i.e. may be NULL.) + // + // Used to make sure all ancestors are also mutable when first + // modifying the container. + CMessage* parent; + + // Pointer to the parent's descriptor that describes this + // field. Used together with the parent's message when making a + // default message instance mutable. + // The pointer is owned by the global DescriptorPool. + const FieldDescriptor* parent_field_descriptor; + const FieldDescriptor* key_field_descriptor; + const FieldDescriptor* value_field_descriptor; + + // A callable that is used to create new child messages. + PyObject* subclass_init; + + // A dict mapping Message* -> CMessage. + PyObject* message_dict; + + // We bump this whenever we perform a mutation, to invalidate existing + // iterators. + uint64 version; +}; + +extern PyTypeObject MessageMapContainer_Type; +extern PyTypeObject MessageMapIterator_Type; + +namespace message_map_container { + +// Builds a MessageMapContainer object, from a parent message and a +// field descriptor. +extern PyObject* NewContainer(CMessage* parent, + const FieldDescriptor* parent_field_descriptor, + PyObject* concrete_class); + +// Releases the messages in the container to a new message. +// +// Returns 0 on success, -1 on failure. +int Release(MessageMapContainer* self); + +// Set the owner field of self and any children of self. +void SetOwner(MessageMapContainer* self, + const shared_ptr<Message>& new_owner); + +} // namespace message_map_container +} // namespace python +} // namespace protobuf + +} // namespace google +#endif // GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_MAP_CONTAINER_H__ diff --git a/python/google/protobuf/pyext/repeated_composite_container.cc b/python/google/protobuf/pyext/repeated_composite_container.cc index 0fe98e73..86b75d0f 100644 --- a/python/google/protobuf/pyext/repeated_composite_container.cc +++ b/python/google/protobuf/pyext/repeated_composite_container.cc @@ -367,8 +367,8 @@ int AssignSubscript(RepeatedCompositeContainer* self, } // Delete from the underlying Message, if any. - if (self->message != NULL) { - if (cmessage::InternalDeleteRepeatedField(self->message, + if (self->parent != NULL) { + if (cmessage::InternalDeleteRepeatedField(self->parent, self->parent_field_descriptor, slice, self->child_messages) < 0) { @@ -572,47 +572,35 @@ static PyObject* Pop(RepeatedCompositeContainer* self, return item; } -// The caller takes ownership of the returned Message. -Message* ReleaseLast(const FieldDescriptor* field, - const Descriptor* type, - Message* message) { +// Release field of parent message and transfer the ownership to target. +void ReleaseLastTo(CMessage* parent, + const FieldDescriptor* field, + CMessage* target) { + GOOGLE_CHECK_NOTNULL(parent); GOOGLE_CHECK_NOTNULL(field); - GOOGLE_CHECK_NOTNULL(type); - GOOGLE_CHECK_NOTNULL(message); + GOOGLE_CHECK_NOTNULL(target); - Message* released_message = message->GetReflection()->ReleaseLast( - message, field); + shared_ptr<Message> released_message( + parent->message->GetReflection()->ReleaseLast(parent->message, field)); // TODO(tibell): Deal with proto1. // ReleaseMessage will return NULL which differs from // child_cmessage->message, if the field does not exist. In this case, // the latter points to the default instance via a const_cast<>, so we // have to reset it to a new mutable object since we are taking ownership. - if (released_message == NULL) { + if (released_message.get() == NULL) { const Message* prototype = - cmessage::GetMessageFactory()->GetPrototype(type); + cmessage::GetMessageFactory()->GetPrototype( + target->message->GetDescriptor()); GOOGLE_CHECK_NOTNULL(prototype); - return prototype->New(); - } else { - return released_message; + released_message.reset(prototype->New()); } -} -// Release field of message and transfer the ownership to cmessage. -void ReleaseLastTo(const FieldDescriptor* field, - Message* message, - CMessage* cmessage) { - GOOGLE_CHECK_NOTNULL(field); - GOOGLE_CHECK_NOTNULL(message); - GOOGLE_CHECK_NOTNULL(cmessage); - - shared_ptr<Message> released_message( - ReleaseLast(field, cmessage->message->GetDescriptor(), message)); - cmessage->parent = NULL; - cmessage->parent_field_descriptor = NULL; - cmessage->message = released_message.get(); - cmessage->read_only = false; - cmessage::SetOwner(cmessage, released_message); + target->parent = NULL; + target->parent_field_descriptor = NULL; + target->message = released_message.get(); + target->read_only = false; + cmessage::SetOwner(target, released_message); } // Called to release a container using @@ -635,7 +623,7 @@ int Release(RepeatedCompositeContainer* self) { for (Py_ssize_t i = size - 1; i >= 0; --i) { CMessage* child_cmessage = reinterpret_cast<CMessage*>( PyList_GET_ITEM(self->child_messages, i)); - ReleaseLastTo(field, message, child_cmessage); + ReleaseLastTo(self->parent, field, child_cmessage); } // Detach from containing message. @@ -732,9 +720,7 @@ static PyMethodDef Methods[] = { PyTypeObject RepeatedCompositeContainer_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - // Keep the fully qualified _message symbol in a line for opensource. - "google.protobuf.pyext._message." - "RepeatedCompositeContainer", // tp_name + FULL_MODULE_NAME ".RepeatedCompositeContainer", // tp_name sizeof(RepeatedCompositeContainer), // tp_basicsize 0, // tp_itemsize (destructor)repeated_composite_container::Dealloc, // tp_dealloc diff --git a/python/google/protobuf/pyext/repeated_composite_container.h b/python/google/protobuf/pyext/repeated_composite_container.h index ce7cee0f..e0f21360 100644 --- a/python/google/protobuf/pyext/repeated_composite_container.h +++ b/python/google/protobuf/pyext/repeated_composite_container.h @@ -161,13 +161,13 @@ int SetOwner(RepeatedCompositeContainer* self, const shared_ptr<Message>& new_owner); // Removes the last element of the repeated message field 'field' on -// the Message 'message', and transfers the ownership of the released -// Message to 'cmessage'. +// the Message 'parent', and transfers the ownership of the released +// Message to 'target'. // // Corresponds to reflection api method ReleaseMessage. -void ReleaseLastTo(const FieldDescriptor* field, - Message* message, - CMessage* cmessage); +void ReleaseLastTo(CMessage* parent, + const FieldDescriptor* field, + CMessage* target); } // namespace repeated_composite_container } // namespace python diff --git a/python/google/protobuf/pyext/repeated_scalar_container.cc b/python/google/protobuf/pyext/repeated_scalar_container.cc index 110a4c85..fd196836 100644 --- a/python/google/protobuf/pyext/repeated_scalar_container.cc +++ b/python/google/protobuf/pyext/repeated_scalar_container.cc @@ -102,7 +102,7 @@ static int AssignItem(RepeatedScalarContainer* self, if (arg == NULL) { ScopedPyObjectPtr py_index(PyLong_FromLong(index)); - return cmessage::InternalDeleteRepeatedField(message, field_descriptor, + return cmessage::InternalDeleteRepeatedField(self->parent, field_descriptor, py_index, NULL); } @@ -470,7 +470,7 @@ static int AssSubscript(RepeatedScalarContainer* self, if (value == NULL) { return cmessage::InternalDeleteRepeatedField( - message, field_descriptor, slice, NULL); + self->parent, field_descriptor, slice, NULL); } if (!create_list) { @@ -769,9 +769,7 @@ static PyMethodDef Methods[] = { PyTypeObject RepeatedScalarContainer_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - // Keep the fully qualified _message symbol in a line for opensource. - "google.protobuf.pyext._message." - "RepeatedScalarContainer", // tp_name + FULL_MODULE_NAME ".RepeatedScalarContainer", // tp_name sizeof(RepeatedScalarContainer), // tp_basicsize 0, // tp_itemsize (destructor)repeated_scalar_container::Dealloc, // tp_dealloc diff --git a/python/google/protobuf/pyext/scalar_map_container.cc b/python/google/protobuf/pyext/scalar_map_container.cc new file mode 100644 index 00000000..6f731d27 --- /dev/null +++ b/python/google/protobuf/pyext/scalar_map_container.cc @@ -0,0 +1,514 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: haberman@google.com (Josh Haberman) + +#include <google/protobuf/pyext/scalar_map_container.h> + +#include <google/protobuf/stubs/common.h> +#include <google/protobuf/message.h> +#include <google/protobuf/pyext/message.h> +#include <google/protobuf/pyext/scoped_pyobject_ptr.h> + +namespace google { +namespace protobuf { +namespace python { + +struct ScalarMapIterator { + PyObject_HEAD; + + // This dict contains the full contents of what we want to iterate over. + // There's no way to avoid building this, because the list representation + // (which is canonical) can contain duplicate keys. So at the very least we + // need a set that lets us skip duplicate keys. And at the point that we're + // doing that, we might as well just build the actual dict we're iterating + // over and use dict's built-in iterator. + PyObject* dict; + + // An iterator on dict. + PyObject* iter; + + // A pointer back to the container, so we can notice changes to the version. + ScalarMapContainer* container; + + // The version of the map when we took the iterator to it. + // + // We store this so that if the map is modified during iteration we can throw + // an error. + uint64 version; +}; + +static ScalarMapIterator* GetIter(PyObject* obj) { + return reinterpret_cast<ScalarMapIterator*>(obj); +} + +namespace scalar_map_container { + +static ScalarMapContainer* GetMap(PyObject* obj) { + return reinterpret_cast<ScalarMapContainer*>(obj); +} + +// The private constructor of ScalarMapContainer objects. +PyObject *NewContainer( + CMessage* parent, const google::protobuf::FieldDescriptor* parent_field_descriptor) { + if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) { + return NULL; + } + + ScopedPyObjectPtr obj(PyType_GenericAlloc(&ScalarMapContainer_Type, 0)); + if (obj.get() == NULL) { + return PyErr_Format(PyExc_RuntimeError, + "Could not allocate new container."); + } + + ScalarMapContainer* self = GetMap(obj); + + self->message = parent->message; + self->parent = parent; + self->parent_field_descriptor = parent_field_descriptor; + self->owner = parent->owner; + self->version = 0; + + self->key_field_descriptor = + parent_field_descriptor->message_type()->FindFieldByName("key"); + self->value_field_descriptor = + parent_field_descriptor->message_type()->FindFieldByName("value"); + + if (self->key_field_descriptor == NULL || + self->value_field_descriptor == NULL) { + return PyErr_Format(PyExc_KeyError, + "Map entry descriptor did not have key/value fields"); + } + + return obj.release(); +} + +// Initializes the underlying Message object of "to" so it becomes a new parent +// repeated scalar, and copies all the values from "from" to it. A child scalar +// container can be released by passing it as both from and to (e.g. making it +// the recipient of the new parent message and copying the values from itself). +static int InitializeAndCopyToParentContainer( + ScalarMapContainer* from, + ScalarMapContainer* to) { + // For now we require from == to, re-evaluate if we want to support deep copy + // as in repeated_scalar_container.cc. + GOOGLE_DCHECK(from == to); + Message* old_message = from->message; + Message* new_message = old_message->New(); + to->parent = NULL; + to->parent_field_descriptor = from->parent_field_descriptor; + to->message = new_message; + to->owner.reset(new_message); + + vector<const FieldDescriptor*> fields; + fields.push_back(from->parent_field_descriptor); + old_message->GetReflection()->SwapFields(old_message, new_message, fields); + return 0; +} + +int Release(ScalarMapContainer* self) { + return InitializeAndCopyToParentContainer(self, self); +} + +void SetOwner(ScalarMapContainer* self, + const shared_ptr<Message>& new_owner) { + self->owner = new_owner; +} + +Py_ssize_t Length(PyObject* _self) { + ScalarMapContainer* self = GetMap(_self); + google::protobuf::Message* message = self->message; + return message->GetReflection()->FieldSize(*message, + self->parent_field_descriptor); +} + +int MapKeyMatches(ScalarMapContainer* self, const Message* entry, + PyObject* key) { + // TODO(haberman): do we need more strict type checking? + ScopedPyObjectPtr entry_key( + cmessage::InternalGetScalar(entry, self->key_field_descriptor)); + int ret = PyObject_RichCompareBool(key, entry_key, Py_EQ); + return ret; +} + +PyObject* GetItem(PyObject* _self, PyObject* key) { + ScalarMapContainer* self = GetMap(_self); + + Message* message = self->message; + const Reflection* reflection = message->GetReflection(); + + // Right now the Reflection API doesn't support map lookup, so we implement it + // via linear search. + // + // TODO(haberman): add lookup API to Reflection API. + size_t size = reflection->FieldSize(*message, self->parent_field_descriptor); + for (int i = size - 1; i >= 0; i--) { + const Message& entry = reflection->GetRepeatedMessage( + *message, self->parent_field_descriptor, i); + int matches = MapKeyMatches(self, &entry, key); + if (matches < 0) return NULL; + if (matches) { + return cmessage::InternalGetScalar(&entry, self->value_field_descriptor); + } + } + + // Need to add a new entry. + Message* entry = + reflection->AddMessage(message, self->parent_field_descriptor); + PyObject* ret = NULL; + + if (cmessage::InternalSetNonOneofScalar(entry, self->key_field_descriptor, + key) >= 0) { + ret = cmessage::InternalGetScalar(entry, self->value_field_descriptor); + } + + self->version++; + + // If there was a type error above, it set the Python exception. + return ret; +} + +int SetItem(PyObject *_self, PyObject *key, PyObject *v) { + ScalarMapContainer* self = GetMap(_self); + cmessage::AssureWritable(self->parent); + + Message* message = self->message; + const Reflection* reflection = message->GetReflection(); + size_t size = + reflection->FieldSize(*message, self->parent_field_descriptor); + self->version++; + + if (v) { + // Set item. + // + // Right now the Reflection API doesn't support map lookup, so we implement + // it via linear search. + // + // TODO(haberman): add lookup API to Reflection API. + for (int i = size - 1; i >= 0; i--) { + Message* entry = reflection->MutableRepeatedMessage( + message, self->parent_field_descriptor, i); + int matches = MapKeyMatches(self, entry, key); + if (matches < 0) return -1; + if (matches) { + return cmessage::InternalSetNonOneofScalar( + entry, self->value_field_descriptor, v); + } + } + + // Key is not already present; insert a new entry. + Message* entry = + reflection->AddMessage(message, self->parent_field_descriptor); + + if (cmessage::InternalSetNonOneofScalar(entry, self->key_field_descriptor, + key) < 0 || + cmessage::InternalSetNonOneofScalar(entry, self->value_field_descriptor, + v) < 0) { + reflection->RemoveLast(message, self->parent_field_descriptor); + return -1; + } + + return 0; + } else { + bool found = false; + for (int i = size - 1; i >= 0; i--) { + Message* entry = reflection->MutableRepeatedMessage( + message, self->parent_field_descriptor, i); + int matches = MapKeyMatches(self, entry, key); + if (matches < 0) return -1; + if (matches) { + found = true; + if (i != size - 1) { + reflection->SwapElements(message, self->parent_field_descriptor, i, + size - 1); + } + reflection->RemoveLast(message, self->parent_field_descriptor); + + // Can't exit now, the repeated field representation of maps allows + // duplicate keys, and we have to be sure to remove all of them. + } + } + + if (found) { + return 0; + } else { + PyErr_Format(PyExc_KeyError, "Key not present in map"); + return -1; + } + } +} + +PyObject* GetIterator(PyObject *_self) { + ScalarMapContainer* self = GetMap(_self); + + ScopedPyObjectPtr obj(PyType_GenericAlloc(&ScalarMapIterator_Type, 0)); + if (obj == NULL) { + return PyErr_Format(PyExc_KeyError, "Could not allocate iterator"); + } + + ScalarMapIterator* iter = GetIter(obj.get()); + + Py_INCREF(self); + iter->container = self; + iter->version = self->version; + iter->dict = PyDict_New(); + if (iter->dict == NULL) { + return PyErr_Format(PyExc_RuntimeError, + "Could not allocate dict for iterator."); + } + + // Build the entire map into a dict right now. Start from the beginning so + // that later entries win in the case of duplicates. + Message* message = self->message; + const Reflection* reflection = message->GetReflection(); + + // Right now the Reflection API doesn't support map lookup, so we implement it + // via linear search. We need to search from the end because the underlying + // representation can have duplicates if a user calls MergeFrom(); the last + // one needs to win. + // + // TODO(haberman): add lookup API to Reflection API. + size_t size = + reflection->FieldSize(*message, self->parent_field_descriptor); + for (int i = 0; i < size; i++) { + Message* entry = reflection->MutableRepeatedMessage( + message, self->parent_field_descriptor, i); + ScopedPyObjectPtr key( + cmessage::InternalGetScalar(entry, self->key_field_descriptor)); + ScopedPyObjectPtr val( + cmessage::InternalGetScalar(entry, self->value_field_descriptor)); + if (PyDict_SetItem(iter->dict, key.get(), val.get()) < 0) { + return PyErr_Format(PyExc_RuntimeError, + "SetItem failed in iterator construction."); + } + } + + + iter->iter = PyObject_GetIter(iter->dict); + + + return obj.release(); +} + +PyObject* Clear(PyObject* _self) { + ScalarMapContainer* self = GetMap(_self); + cmessage::AssureWritable(self->parent); + Message* message = self->message; + const Reflection* reflection = message->GetReflection(); + + reflection->ClearField(message, self->parent_field_descriptor); + + Py_RETURN_NONE; +} + +PyObject* Contains(PyObject* _self, PyObject* key) { + ScalarMapContainer* self = GetMap(_self); + + Message* message = self->message; + const Reflection* reflection = message->GetReflection(); + + // Right now the Reflection API doesn't support map lookup, so we implement it + // via linear search. + // + // TODO(haberman): add lookup API to Reflection API. + size_t size = reflection->FieldSize(*message, self->parent_field_descriptor); + for (int i = size - 1; i >= 0; i--) { + const Message& entry = reflection->GetRepeatedMessage( + *message, self->parent_field_descriptor, i); + int matches = MapKeyMatches(self, &entry, key); + if (matches < 0) return NULL; + if (matches) { + Py_RETURN_TRUE; + } + } + + Py_RETURN_FALSE; +} + +PyObject* Get(PyObject* self, PyObject* args) { + PyObject* key; + PyObject* default_value = NULL; + if (PyArg_ParseTuple(args, "O|O", &key, &default_value) < 0) { + return NULL; + } + + ScopedPyObjectPtr is_present(Contains(self, key)); + if (is_present.get() == NULL) { + return NULL; + } + + if (PyObject_IsTrue(is_present.get())) { + return GetItem(self, key); + } else { + if (default_value != NULL) { + Py_INCREF(default_value); + return default_value; + } else { + Py_RETURN_NONE; + } + } +} + +static PyMappingMethods MpMethods = { + Length, // mp_length + GetItem, // mp_subscript + SetItem, // mp_ass_subscript +}; + +static void Dealloc(PyObject* _self) { + ScalarMapContainer* self = GetMap(_self); + self->owner.reset(); + Py_TYPE(_self)->tp_free(_self); +} + +static PyMethodDef Methods[] = { + { "__contains__", Contains, METH_O, + "Tests whether a key is a member of the map." }, + { "clear", (PyCFunction)Clear, METH_NOARGS, + "Removes all elements from the map." }, + { "get", Get, METH_VARARGS, + "Gets the value for the given key if present, or otherwise a default" }, + /* + { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS, + "Makes a deep copy of the class." }, + { "__reduce__", (PyCFunction)Reduce, METH_NOARGS, + "Outputs picklable representation of the repeated field." }, + */ + {NULL, NULL}, +}; + +} // namespace scalar_map_container + +namespace scalar_map_iterator { + +static void Dealloc(PyObject* _self) { + ScalarMapIterator* self = GetIter(_self); + Py_DECREF(self->dict); + Py_DECREF(self->iter); + Py_DECREF(self->container); + Py_TYPE(_self)->tp_free(_self); +} + +PyObject* IterNext(PyObject* _self) { + ScalarMapIterator* self = GetIter(_self); + + // This won't catch mutations to the map performed by MergeFrom(); no easy way + // to address that. + if (self->version != self->container->version) { + return PyErr_Format(PyExc_RuntimeError, + "Map modified during iteration."); + } + + return PyIter_Next(self->iter); +} + +} // namespace scalar_map_iterator + +PyTypeObject ScalarMapContainer_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + FULL_MODULE_NAME ".ScalarMapContainer", // tp_name + sizeof(ScalarMapContainer), // tp_basicsize + 0, // tp_itemsize + scalar_map_container::Dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + &scalar_map_container::MpMethods, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "A scalar map container", // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + scalar_map_container::GetIterator, // tp_iter + 0, // tp_iternext + scalar_map_container::Methods, // tp_methods + 0, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init +}; + +PyTypeObject ScalarMapIterator_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + FULL_MODULE_NAME ".ScalarMapIterator", // tp_name + sizeof(ScalarMapIterator), // tp_basicsize + 0, // tp_itemsize + scalar_map_iterator::Dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "A scalar map iterator", // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + PyObject_SelfIter, // tp_iter + scalar_map_iterator::IterNext, // tp_iternext + 0, // tp_methods + 0, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init +}; + +} // namespace python +} // namespace protobuf +} // namespace google diff --git a/python/google/protobuf/pyext/scalar_map_container.h b/python/google/protobuf/pyext/scalar_map_container.h new file mode 100644 index 00000000..254e6e98 --- /dev/null +++ b/python/google/protobuf/pyext/scalar_map_container.h @@ -0,0 +1,110 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_SCALAR_MAP_CONTAINER_H__ +#define GOOGLE_PROTOBUF_PYTHON_CPP_SCALAR_MAP_CONTAINER_H__ + +#include <Python.h> + +#include <memory> +#ifndef _SHARED_PTR_H +#include <google/protobuf/stubs/shared_ptr.h> +#endif + +#include <google/protobuf/descriptor.h> + +namespace google { +namespace protobuf { + +class Message; + +using internal::shared_ptr; + +namespace python { + +struct CMessage; + +struct ScalarMapContainer { + PyObject_HEAD; + + // This is the top-level C++ Message object that owns the whole + // proto tree. Every Python ScalarMapContainer holds a + // reference to it in order to keep it alive as long as there's a + // Python object that references any part of the tree. + shared_ptr<Message> owner; + + // Pointer to the C++ Message that contains this container. The + // ScalarMapContainer does not own this pointer. + Message* message; + + // Weak reference to a parent CMessage object (i.e. may be NULL.) + // + // Used to make sure all ancestors are also mutable when first + // modifying the container. + CMessage* parent; + + // Pointer to the parent's descriptor that describes this + // field. Used together with the parent's message when making a + // default message instance mutable. + // The pointer is owned by the global DescriptorPool. + const FieldDescriptor* parent_field_descriptor; + const FieldDescriptor* key_field_descriptor; + const FieldDescriptor* value_field_descriptor; + + // We bump this whenever we perform a mutation, to invalidate existing + // iterators. + uint64 version; +}; + +extern PyTypeObject ScalarMapContainer_Type; +extern PyTypeObject ScalarMapIterator_Type; + +namespace scalar_map_container { + +// Builds a ScalarMapContainer object, from a parent message and a +// field descriptor. +extern PyObject *NewContainer( + CMessage* parent, const FieldDescriptor* parent_field_descriptor); + +// Releases the messages in the container to a new message. +// +// Returns 0 on success, -1 on failure. +int Release(ScalarMapContainer* self); + +// Set the owner field of self and any children of self. +void SetOwner(ScalarMapContainer* self, + const shared_ptr<Message>& new_owner); + +} // namespace scalar_map_container +} // namespace python +} // namespace protobuf + +} // namespace google +#endif // GOOGLE_PROTOBUF_PYTHON_CPP_SCALAR_MAP_CONTAINER_H__ |