diff options
Diffstat (limited to 'python/google/protobuf/pyext')
27 files changed, 1406 insertions, 508 deletions
diff --git a/python/google/protobuf/pyext/descriptor.cc b/python/google/protobuf/pyext/descriptor.cc index 8af0cb12..3cb16b74 100644 --- a/python/google/protobuf/pyext/descriptor.cc +++ b/python/google/protobuf/pyext/descriptor.cc @@ -32,8 +32,8 @@ #include <Python.h> #include <frameobject.h> -#include <google/protobuf/stubs/hash.h> #include <string> +#include <unordered_map> #include <google/protobuf/io/coded_stream.h> #include <google/protobuf/descriptor.pb.h> @@ -44,6 +44,7 @@ #include <google/protobuf/pyext/message.h> #include <google/protobuf/pyext/message_factory.h> #include <google/protobuf/pyext/scoped_pyobject_ptr.h> +#include <google/protobuf/stubs/hash.h> #if PY_MAJOR_VERSION >= 3 #define PyString_FromStringAndSize PyUnicode_FromStringAndSize @@ -54,10 +55,12 @@ #if PY_VERSION_HEX < 0x03030000 #error "Python 3.0 - 3.2 are not supported." #endif - #define PyString_AsStringAndSize(ob, charpp, sizep) \ - (PyUnicode_Check(ob)? \ - ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \ - PyBytes_AsStringAndSize(ob, (charpp), (sizep))) +#define PyString_AsStringAndSize(ob, charpp, sizep) \ + (PyUnicode_Check(ob) ? ((*(charpp) = const_cast<char*>( \ + PyUnicode_AsUTF8AndSize(ob, (sizep)))) == NULL \ + ? -1 \ + : 0) \ + : PyBytes_AsStringAndSize(ob, (charpp), (sizep))) #endif namespace google { @@ -70,7 +73,7 @@ namespace python { // released. // This is enough to support the "is" operator on live objects. // All descriptors are stored here. -hash_map<const void*, PyObject*> interned_descriptors; +std::unordered_map<const void*, PyObject*>* interned_descriptors; PyObject* PyString_FromCppString(const string& str) { return PyString_FromStringAndSize(str.c_str(), str.size()); @@ -119,8 +122,10 @@ bool _CalledFromGeneratedFile(int stacklevel) { PyErr_Clear(); return false; } - if ((filename_size < 3) || (strcmp(&filename[filename_size - 3], ".py") != 0)) { - // Cython's stack does not have .py file name and is not at global module scope. + if ((filename_size < 3) || + (strcmp(&filename[filename_size - 3], ".py") != 0)) { + // Cython's stack does not have .py file name and is not at global module + // scope. return true; } if (filename_size < 7) { @@ -131,7 +136,7 @@ bool _CalledFromGeneratedFile(int stacklevel) { // Filename is not ending with _pb2. return false; } - + if (frame->f_globals != frame->f_locals) { // Not at global module scope return false; @@ -197,7 +202,7 @@ static PyObject* GetOrBuildOptions(const DescriptorClass *descriptor) { // First search in the cache. PyDescriptorPool* caching_pool = GetDescriptorPool_FromPool( GetFileDescriptor(descriptor)->pool()); - hash_map<const void*, PyObject*>* descriptor_options = + std::unordered_map<const void*, PyObject*>* descriptor_options = caching_pool->descriptor_options; if (descriptor_options->find(descriptor) != descriptor_options->end()) { PyObject *value = (*descriptor_options)[descriptor]; @@ -232,7 +237,7 @@ static PyObject* GetOrBuildOptions(const DescriptorClass *descriptor) { if (value == NULL) { return NULL; } - if (!PyObject_TypeCheck(value.get(), &CMessage_Type)) { + if (!PyObject_TypeCheck(value.get(), CMessage_Type)) { PyErr_Format(PyExc_TypeError, "Invalid class for %s: %s", message_type->full_name().c_str(), Py_TYPE(value.get())->tp_name); @@ -275,7 +280,7 @@ static PyObject* CopyToPythonProto(const DescriptorClass *descriptor, const Descriptor* self_descriptor = DescriptorProtoClass::default_instance().GetDescriptor(); CMessage* message = reinterpret_cast<CMessage*>(target); - if (!PyObject_TypeCheck(target, &CMessage_Type) || + if (!PyObject_TypeCheck(target, CMessage_Type) || message->message->GetDescriptor() != self_descriptor) { PyErr_Format(PyExc_TypeError, "Not a %s message", self_descriptor->full_name().c_str()); @@ -332,9 +337,9 @@ PyObject* NewInternedDescriptor(PyTypeObject* type, } // See if the object is in the map of interned descriptors - hash_map<const void*, PyObject*>::iterator it = - interned_descriptors.find(descriptor); - if (it != interned_descriptors.end()) { + std::unordered_map<const void*, PyObject*>::iterator it = + interned_descriptors->find(descriptor); + if (it != interned_descriptors->end()) { GOOGLE_DCHECK(Py_TYPE(it->second) == type); Py_INCREF(it->second); return it->second; @@ -348,7 +353,7 @@ PyObject* NewInternedDescriptor(PyTypeObject* type, py_descriptor->descriptor = descriptor; // and cache it. - interned_descriptors.insert( + interned_descriptors->insert( std::make_pair(descriptor, reinterpret_cast<PyObject*>(py_descriptor))); // Ensures that the DescriptorPool stays alive. @@ -370,7 +375,7 @@ PyObject* NewInternedDescriptor(PyTypeObject* type, static void Dealloc(PyBaseDescriptor* self) { // Remove from interned dictionary - interned_descriptors.erase(self->descriptor); + interned_descriptors->erase(self->descriptor); Py_CLEAR(self->pool); Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self)); } @@ -758,6 +763,11 @@ static PyObject* HasDefaultValue(PyBaseDescriptor *self, void *closure) { static PyObject* GetDefaultValue(PyBaseDescriptor *self, void *closure) { PyObject *result; + if (_GetDescriptor(self)->is_repeated()) { + return PyList_New(0); + } + + switch (_GetDescriptor(self)->cpp_type()) { case FieldDescriptor::CPPTYPE_INT32: { int32 value = _GetDescriptor(self)->default_value_int32(); @@ -805,6 +815,10 @@ static PyObject* GetDefaultValue(PyBaseDescriptor *self, void *closure) { result = PyInt_FromLong(value->number()); break; } + case FieldDescriptor::CPPTYPE_MESSAGE: { + Py_RETURN_NONE; + break; + } default: PyErr_Format(PyExc_NotImplementedError, "default value for %s", _GetDescriptor(self)->full_name().c_str()); @@ -1919,6 +1933,9 @@ bool InitDescriptor() { if (!InitDescriptorMappingTypes()) return false; + // Initialize globals defined in this file. + interned_descriptors = new std::unordered_map<const void*, PyObject*>; + return true; } diff --git a/python/google/protobuf/pyext/descriptor.h b/python/google/protobuf/pyext/descriptor.h index f081df84..c4dde9e7 100644 --- a/python/google/protobuf/pyext/descriptor.h +++ b/python/google/protobuf/pyext/descriptor.h @@ -100,6 +100,6 @@ bool InitDescriptor(); } // namespace python } // namespace protobuf - } // namespace google + #endif // GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_H__ diff --git a/python/google/protobuf/pyext/descriptor_containers.cc b/python/google/protobuf/pyext/descriptor_containers.cc index bc007f7e..d5b5dc68 100644 --- a/python/google/protobuf/pyext/descriptor_containers.cc +++ b/python/google/protobuf/pyext/descriptor_containers.cc @@ -33,7 +33,7 @@ // // They avoid the allocation of a full dictionary or a full list: they simply // store a pointer to the parent descriptor, use the C++ Descriptor methods (see -// google/protobuf/descriptor.h) to retrieve other descriptors, and create +// net/proto2/public/descriptor.h) to retrieve other descriptors, and create // Python objects on the fly. // // The containers fully conform to abc.Mapping and abc.Sequence, and behave just @@ -64,10 +64,12 @@ #if PY_VERSION_HEX < 0x03030000 #error "Python 3.0 - 3.2 are not supported." #endif - #define PyString_AsStringAndSize(ob, charpp, sizep) \ - (PyUnicode_Check(ob)? \ - ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \ - PyBytes_AsStringAndSize(ob, (charpp), (sizep))) +#define PyString_AsStringAndSize(ob, charpp, sizep) \ + (PyUnicode_Check(ob) ? ((*(charpp) = const_cast<char*>( \ + PyUnicode_AsUTF8AndSize(ob, (sizep)))) == NULL \ + ? -1 \ + : 0) \ + : PyBytes_AsStringAndSize(ob, (charpp), (sizep))) #endif namespace google { diff --git a/python/google/protobuf/pyext/descriptor_containers.h b/python/google/protobuf/pyext/descriptor_containers.h index 83de07b6..4e05c58e 100644 --- a/python/google/protobuf/pyext/descriptor_containers.h +++ b/python/google/protobuf/pyext/descriptor_containers.h @@ -104,6 +104,6 @@ PyObject* NewServiceMethodsByName(const ServiceDescriptor* descriptor); } // namespace python } // namespace protobuf - } // namespace google + #endif // GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_CONTAINERS_H__ diff --git a/python/google/protobuf/pyext/descriptor_database.cc b/python/google/protobuf/pyext/descriptor_database.cc index daa40cc7..0514b35c 100644 --- a/python/google/protobuf/pyext/descriptor_database.cc +++ b/python/google/protobuf/pyext/descriptor_database.cc @@ -70,7 +70,7 @@ static bool GetFileDescriptorProto(PyObject* py_descriptor, const Descriptor* filedescriptor_descriptor = FileDescriptorProto::default_instance().GetDescriptor(); CMessage* message = reinterpret_cast<CMessage*>(py_descriptor); - if (PyObject_TypeCheck(py_descriptor, &CMessage_Type) && + if (PyObject_TypeCheck(py_descriptor, CMessage_Type) && message->message->GetDescriptor() == filedescriptor_descriptor) { // Fast path: Just use the pointer. FileDescriptorProto* file_proto = @@ -143,6 +143,43 @@ bool PyDescriptorDatabase::FindFileContainingExtension( return GetFileDescriptorProto(py_descriptor.get(), output); } +// Finds the tag numbers used by all known extensions of +// containing_type, and appends them to output in an undefined +// order. +// Python DescriptorDatabases are not required to implement this method. +bool PyDescriptorDatabase::FindAllExtensionNumbers( + const string& containing_type, std::vector<int>* output) { + ScopedPyObjectPtr py_method( + PyObject_GetAttrString(py_database_, "FindAllExtensionNumbers")); + if (py_method == NULL) { + // This method is not implemented, returns without error. + PyErr_Clear(); + return false; + } + ScopedPyObjectPtr py_list( + PyObject_CallFunction(py_method.get(), "s#", containing_type.c_str(), + containing_type.size())); + if (py_list == NULL) { + PyErr_Print(); + return false; + } + Py_ssize_t size = PyList_Size(py_list.get()); + int64 item_value; + for (Py_ssize_t i = 0 ; i < size; ++i) { + ScopedPyObjectPtr item(PySequence_GetItem(py_list.get(), i)); + item_value = PyLong_AsLong(item.get()); + if (item_value < 0) { + GOOGLE_LOG(ERROR) + << "FindAllExtensionNumbers method did not return " + << "valid extension numbers."; + PyErr_Print(); + return false; + } + output->push_back(item_value); + } + return true; +} + } // namespace python } // namespace protobuf } // namespace google diff --git a/python/google/protobuf/pyext/descriptor_database.h b/python/google/protobuf/pyext/descriptor_database.h index fc71c4bc..30aa1b73 100644 --- a/python/google/protobuf/pyext/descriptor_database.h +++ b/python/google/protobuf/pyext/descriptor_database.h @@ -48,21 +48,28 @@ class PyDescriptorDatabase : public DescriptorDatabase { // with a copy of FileDescriptorProto. // Find a file by file name. - bool FindFileByName(const string& filename, + bool FindFileByName(const std::string& filename, FileDescriptorProto* output); // Find the file that declares the given fully-qualified symbol name. - bool FindFileContainingSymbol(const string& symbol_name, + bool FindFileContainingSymbol(const std::string& symbol_name, FileDescriptorProto* output); // Find the file which defines an extension extending the given message type // with the given field number. // Containing_type must be a fully-qualified type name. // Python objects are not required to implement this method. - bool FindFileContainingExtension(const string& containing_type, + bool FindFileContainingExtension(const std::string& containing_type, int field_number, FileDescriptorProto* output); + // Finds the tag numbers used by all known extensions of + // containing_type, and appends them to output in an undefined + // order. + // Python objects are not required to implement this method. + bool FindAllExtensionNumbers(const std::string& containing_type, + std::vector<int>* output); + private: // The python object that implements the database. The reference is owned. PyObject* py_database_; @@ -70,6 +77,6 @@ class PyDescriptorDatabase : public DescriptorDatabase { } // namespace python } // namespace protobuf - } // namespace google + #endif // GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_DATABASE_H__ diff --git a/python/google/protobuf/pyext/descriptor_pool.cc b/python/google/protobuf/pyext/descriptor_pool.cc index 95882aeb..d0038b10 100644 --- a/python/google/protobuf/pyext/descriptor_pool.cc +++ b/python/google/protobuf/pyext/descriptor_pool.cc @@ -30,6 +30,8 @@ // Implements the DescriptorPool, which collects all descriptors. +#include <unordered_map> + #include <Python.h> #include <google/protobuf/descriptor.pb.h> @@ -46,10 +48,12 @@ #if PY_VERSION_HEX < 0x03030000 #error "Python 3.0 - 3.2 are not supported." #endif - #define PyString_AsStringAndSize(ob, charpp, sizep) \ - (PyUnicode_Check(ob)? \ - ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \ - PyBytes_AsStringAndSize(ob, (charpp), (sizep))) +#define PyString_AsStringAndSize(ob, charpp, sizep) \ + (PyUnicode_Check(ob) ? ((*(charpp) = const_cast<char*>( \ + PyUnicode_AsUTF8AndSize(ob, (sizep)))) == NULL \ + ? -1 \ + : 0) \ + : PyBytes_AsStringAndSize(ob, (charpp), (sizep))) #endif namespace google { @@ -58,7 +62,8 @@ namespace python { // A map to cache Python Pools per C++ pointer. // Pointers are not owned here, and belong to the PyDescriptorPool. -static hash_map<const DescriptorPool*, PyDescriptorPool*> descriptor_pool_map; +static std::unordered_map<const DescriptorPool*, PyDescriptorPool*>* + descriptor_pool_map; namespace cdescriptor_pool { @@ -74,8 +79,7 @@ static PyDescriptorPool* _CreateDescriptorPool() { cpool->underlay = NULL; cpool->database = NULL; - cpool->descriptor_options = - new hash_map<const void*, PyObject *>(); + cpool->descriptor_options = new std::unordered_map<const void*, PyObject*>(); cpool->py_message_factory = message_factory::NewMessageFactory( &PyMessageFactory_Type, cpool); @@ -101,7 +105,7 @@ static PyDescriptorPool* PyDescriptorPool_NewWithUnderlay( cpool->pool = new DescriptorPool(underlay); cpool->underlay = underlay; - if (!descriptor_pool_map.insert( + if (!descriptor_pool_map->insert( std::make_pair(cpool->pool, cpool)).second) { // Should never happen -- would indicate an internal error / bug. PyErr_SetString(PyExc_ValueError, "DescriptorPool already registered"); @@ -124,7 +128,7 @@ static PyDescriptorPool* PyDescriptorPool_NewWithDatabase( cpool->pool = new DescriptorPool(); } - if (!descriptor_pool_map.insert(std::make_pair(cpool->pool, cpool)).second) { + if (!descriptor_pool_map->insert(std::make_pair(cpool->pool, cpool)).second) { // Should never happen -- would indicate an internal error / bug. PyErr_SetString(PyExc_ValueError, "DescriptorPool already registered"); return NULL; @@ -151,9 +155,9 @@ static PyObject* New(PyTypeObject* type, static void Dealloc(PyObject* pself) { PyDescriptorPool* self = reinterpret_cast<PyDescriptorPool*>(pself); - descriptor_pool_map.erase(self->pool); + descriptor_pool_map->erase(self->pool); Py_CLEAR(self->py_message_factory); - for (hash_map<const void*, PyObject*>::iterator it = + for (std::unordered_map<const void*, PyObject*>::iterator it = self->descriptor_options->begin(); it != self->descriptor_options->end(); ++it) { Py_DECREF(it->second); @@ -180,6 +184,7 @@ static PyObject* FindMessageByName(PyObject* self, PyObject* arg) { return NULL; } + return PyMessageDescriptor_FromDescriptor(message_descriptor); } @@ -218,6 +223,7 @@ PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* arg) { return NULL; } + return PyFieldDescriptor_FromDescriptor(field_descriptor); } @@ -239,6 +245,7 @@ PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg) { return NULL; } + return PyFieldDescriptor_FromDescriptor(field_descriptor); } @@ -260,6 +267,7 @@ PyObject* FindEnumTypeByName(PyDescriptorPool* self, PyObject* arg) { return NULL; } + return PyEnumDescriptor_FromDescriptor(enum_descriptor); } @@ -281,6 +289,7 @@ PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg) { return NULL; } + return PyOneofDescriptor_FromDescriptor(oneof_descriptor); } @@ -303,6 +312,7 @@ static PyObject* FindServiceByName(PyObject* self, PyObject* arg) { return NULL; } + return PyServiceDescriptor_FromDescriptor(service_descriptor); } @@ -321,6 +331,7 @@ static PyObject* FindMethodByName(PyObject* self, PyObject* arg) { return NULL; } + return PyMethodDescriptor_FromDescriptor(method_descriptor); } @@ -339,6 +350,7 @@ static PyObject* FindFileContainingSymbol(PyObject* self, PyObject* arg) { return NULL; } + return PyFileDescriptor_FromDescriptor(file_descriptor); } @@ -362,6 +374,7 @@ static PyObject* FindExtensionByNumber(PyObject* self, PyObject* args) { return NULL; } + return PyFieldDescriptor_FromDescriptor(extension_descriptor); } @@ -668,13 +681,17 @@ bool InitDescriptorPool() { // The Pool of messages declared in Python libraries. // generated_pool() contains all messages already linked in C++ libraries, and // is used as underlay. + descriptor_pool_map = + new std::unordered_map<const DescriptorPool*, PyDescriptorPool*>; python_generated_pool = cdescriptor_pool::PyDescriptorPool_NewWithUnderlay( DescriptorPool::generated_pool()); if (python_generated_pool == NULL) { + delete descriptor_pool_map; return false; } + // Register this pool to be found for C++-generated descriptors. - descriptor_pool_map.insert( + descriptor_pool_map->insert( std::make_pair(DescriptorPool::generated_pool(), python_generated_pool)); @@ -695,9 +712,9 @@ PyDescriptorPool* GetDescriptorPool_FromPool(const DescriptorPool* pool) { pool == DescriptorPool::generated_pool()) { return python_generated_pool; } - hash_map<const DescriptorPool*, PyDescriptorPool*>::iterator it = - descriptor_pool_map.find(pool); - if (it == descriptor_pool_map.end()) { + std::unordered_map<const DescriptorPool*, PyDescriptorPool*>::iterator it = + descriptor_pool_map->find(pool); + if (it == descriptor_pool_map->end()) { PyErr_SetString(PyExc_KeyError, "Unknown descriptor pool"); return NULL; } diff --git a/python/google/protobuf/pyext/descriptor_pool.h b/python/google/protobuf/pyext/descriptor_pool.h index 53ee53dc..8e7b4d6b 100644 --- a/python/google/protobuf/pyext/descriptor_pool.h +++ b/python/google/protobuf/pyext/descriptor_pool.h @@ -33,7 +33,7 @@ #include <Python.h> -#include <google/protobuf/stubs/hash.h> +#include <unordered_map> #include <google/protobuf/descriptor.h> namespace google { @@ -77,7 +77,7 @@ typedef struct PyDescriptorPool { // Cache the options for any kind of descriptor. // Descriptor pointers are owned by the DescriptorPool above. // Python objects are owned by the map. - hash_map<const void*, PyObject*>* descriptor_options; + std::unordered_map<const void*, PyObject*>* descriptor_options; } PyDescriptorPool; @@ -89,7 +89,7 @@ namespace cdescriptor_pool { // Looks up a message by name. // Returns a message Descriptor, or NULL if not found. const Descriptor* FindMessageTypeByName(PyDescriptorPool* self, - const string& name); + const std::string& name); // The functions below are also exposed as methods of the DescriptorPool type. @@ -140,6 +140,6 @@ bool InitDescriptorPool(); } // namespace python } // namespace protobuf - } // namespace google + #endif // GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_POOL_H__ diff --git a/python/google/protobuf/pyext/extension_dict.cc b/python/google/protobuf/pyext/extension_dict.cc index 018b5c2c..b73368eb 100644 --- a/python/google/protobuf/pyext/extension_dict.cc +++ b/python/google/protobuf/pyext/extension_dict.cc @@ -51,10 +51,12 @@ #if PY_VERSION_HEX < 0x03030000 #error "Python 3.0 - 3.2 are not supported." #endif - #define PyString_AsStringAndSize(ob, charpp, sizep) \ - (PyUnicode_Check(ob)? \ - ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \ - PyBytes_AsStringAndSize(ob, (charpp), (sizep))) +#define PyString_AsStringAndSize(ob, charpp, sizep) \ + (PyUnicode_Check(ob) ? ((*(charpp) = const_cast<char*>( \ + PyUnicode_AsUTF8AndSize(ob, (sizep)))) == NULL \ + ? -1 \ + : 0) \ + : PyBytes_AsStringAndSize(ob, (charpp), (sizep))) #endif namespace google { @@ -63,40 +65,25 @@ namespace python { namespace extension_dict { -PyObject* len(ExtensionDict* self) { -#if PY_MAJOR_VERSION >= 3 - return PyLong_FromLong(PyDict_Size(self->values)); -#else - return PyInt_FromLong(PyDict_Size(self->values)); -#endif -} - PyObject* subscript(ExtensionDict* self, PyObject* key) { const FieldDescriptor* descriptor = cmessage::GetExtensionDescriptor(key); if (descriptor == NULL) { return NULL; } - if (!CheckFieldBelongsToMessage(descriptor, self->message)) { + if (!CheckFieldBelongsToMessage(descriptor, self->parent->message)) { return NULL; } if (descriptor->label() != FieldDescriptor::LABEL_REPEATED && descriptor->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) { - return cmessage::InternalGetScalar(self->message, descriptor); + return cmessage::InternalGetScalar(self->parent->message, descriptor); } - PyObject* value = PyDict_GetItem(self->values, key); - if (value != NULL) { - Py_INCREF(value); - return value; - } - - if (self->parent == NULL) { - // We are in "detached" state. Don't allow further modifications. - // TODO(amauryfa): Support adding non-scalars to a detached extension dict. - // This probably requires to store the type of the main message. - PyErr_SetObject(PyExc_KeyError, key); - return NULL; + CMessage::CompositeFieldsMap::iterator iterator = + self->parent->composite_fields->find(descriptor); + if (iterator != self->parent->composite_fields->end()) { + Py_INCREF(iterator->second); + return iterator->second; } if (descriptor->label() != FieldDescriptor::LABEL_REPEATED && @@ -107,7 +94,8 @@ PyObject* subscript(ExtensionDict* self, PyObject* key) { if (sub_message == NULL) { return NULL; } - PyDict_SetItem(self->values, key, sub_message); + Py_INCREF(sub_message); + (*self->parent->composite_fields)[descriptor] = sub_message; return sub_message; } @@ -136,7 +124,8 @@ PyObject* subscript(ExtensionDict* self, PyObject* key) { if (py_container == NULL) { return NULL; } - PyDict_SetItem(self->values, key, py_container); + Py_INCREF(py_container); + (*self->parent->composite_fields)[descriptor] = py_container; return py_container; } else { PyObject* py_container = repeated_scalar_container::NewContainer( @@ -144,7 +133,8 @@ PyObject* subscript(ExtensionDict* self, PyObject* key) { if (py_container == NULL) { return NULL; } - PyDict_SetItem(self->values, key, py_container); + Py_INCREF(py_container); + (*self->parent->composite_fields)[descriptor] = py_container; return py_container; } } @@ -157,7 +147,7 @@ int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value) { if (descriptor == NULL) { return -1; } - if (!CheckFieldBelongsToMessage(descriptor, self->message)) { + if (!CheckFieldBelongsToMessage(descriptor, self->parent->message)) { return -1; } @@ -167,14 +157,10 @@ int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value) { "type"); return -1; } - if (self->parent) { - cmessage::AssureWritable(self->parent); - if (cmessage::InternalSetScalar(self->parent, descriptor, value) < 0) { - return -1; - } + cmessage::AssureWritable(self->parent); + if (cmessage::InternalSetScalar(self->parent, descriptor, value) < 0) { + return -1; } - // TODO(tibell): We shouldn't write scalars to the cache. - PyDict_SetItem(self->values, key, value); return 0; } @@ -232,22 +218,36 @@ ExtensionDict* NewExtensionDict(CMessage *parent) { return NULL; } - self->parent = parent; // Store a borrowed reference. - self->message = parent->message; - self->owner = parent->owner; - self->values = PyDict_New(); + Py_INCREF(parent); + self->parent = parent; return self; } void dealloc(ExtensionDict* self) { - Py_CLEAR(self->values); - self->owner.reset(); + Py_CLEAR(self->parent); Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self)); } +static PyObject* RichCompare(ExtensionDict* self, PyObject* other, int opid) { + // Only equality comparisons are implemented. + if (opid != Py_EQ && opid != Py_NE) { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + bool equals = false; + if (PyObject_TypeCheck(other, &ExtensionDict_Type)) { + equals = self->parent == reinterpret_cast<ExtensionDict*>(other)->parent;; + } + if (equals ^ (opid == Py_EQ)) { + Py_RETURN_FALSE; + } else { + Py_RETURN_TRUE; + } +} + static PyMappingMethods MpMethods = { - (lenfunc)len, /* mp_length */ - (binaryfunc)subscript, /* mp_subscript */ + (lenfunc)NULL, /* mp_length */ + (binaryfunc)subscript, /* mp_subscript */ (objobjargproc)ass_subscript,/* mp_ass_subscript */ }; @@ -286,7 +286,7 @@ PyTypeObject ExtensionDict_Type = { "An extension dict", // tp_doc 0, // tp_traverse 0, // tp_clear - 0, // tp_richcompare + (richcmpfunc)extension_dict::RichCompare, // tp_richcompare 0, // tp_weaklistoffset 0, // tp_iter 0, // tp_iternext diff --git a/python/google/protobuf/pyext/extension_dict.h b/python/google/protobuf/pyext/extension_dict.h index 0de2c4ee..a7d6bb7b 100644 --- a/python/google/protobuf/pyext/extension_dict.h +++ b/python/google/protobuf/pyext/extension_dict.h @@ -51,23 +51,8 @@ namespace python { typedef struct ExtensionDict { PyObject_HEAD; - // This is the top-level C++ Message object that owns the whole - // proto tree. Every Python container class holds a - // reference to it in order to keep it alive as long as there's a - // Python object that references any part of the tree. - CMessage::OwnerRef owner; - - // Weak reference to parent message. Used to make sure - // the parent is writable when an extension field is modified. + // Strong, owned reference to the parent message. Never NULL. CMessage* parent; - - // Pointer to the C++ Message that this ExtensionDict extends. - // Not owned by us. - Message* message; - - // A dict of child messages, indexed by Extension descriptors. - // Similar to CMessage::composite_fields. - PyObject* values; } ExtensionDict; extern PyTypeObject ExtensionDict_Type; @@ -80,6 +65,6 @@ ExtensionDict* NewExtensionDict(CMessage *parent); } // namespace extension_dict } // namespace python } // namespace protobuf - } // namespace google + #endif // GOOGLE_PROTOBUF_PYTHON_CPP_EXTENSION_DICT_H__ diff --git a/python/google/protobuf/pyext/field.cc b/python/google/protobuf/pyext/field.cc new file mode 100755 index 00000000..1afd4583 --- /dev/null +++ b/python/google/protobuf/pyext/field.cc @@ -0,0 +1,142 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <google/protobuf/pyext/field.h> + +#include <google/protobuf/descriptor.h> +#include <google/protobuf/pyext/descriptor.h> +#include <google/protobuf/pyext/message.h> + +#if PY_MAJOR_VERSION >= 3 + #define PyString_FromFormat PyUnicode_FromFormat +#endif + +namespace google { +namespace protobuf { +namespace python { + +namespace field { + +static PyObject* Repr(PyMessageFieldProperty* self) { + return PyString_FromFormat("<field property '%s'>", + self->field_descriptor->full_name().c_str()); +} + +static PyObject* DescrGet(PyMessageFieldProperty* self, PyObject* obj, + PyObject* type) { + if (obj == NULL) { + Py_INCREF(self); + return reinterpret_cast<PyObject*>(self); + } + return cmessage::GetFieldValue(reinterpret_cast<CMessage*>(obj), + self->field_descriptor); +} + +static int DescrSet(PyMessageFieldProperty* self, PyObject* obj, + PyObject* value) { + if (value == NULL) { + PyErr_SetString(PyExc_AttributeError, "Cannot delete field attribute"); + return -1; + } + return cmessage::SetFieldValue(reinterpret_cast<CMessage*>(obj), + self->field_descriptor, value); +} + +static PyObject* GetDescriptor(PyMessageFieldProperty* self, void* closure) { + return PyFieldDescriptor_FromDescriptor(self->field_descriptor); +} + +static PyObject* GetDoc(PyMessageFieldProperty* self, void* closure) { + return PyString_FromFormat("Field %s", + self->field_descriptor->full_name().c_str()); +} + +static PyGetSetDef Getters[] = { + {"DESCRIPTOR", (getter)GetDescriptor, NULL, "Field descriptor"}, + {"__doc__", (getter)GetDoc, NULL, NULL}, + {NULL}}; +} // namespace field + +static PyTypeObject _CFieldProperty_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) // head + FULL_MODULE_NAME ".FieldProperty", // tp_name + sizeof(PyMessageFieldProperty), // tp_basicsize + 0, // tp_itemsize + 0, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + (reprfunc)field::Repr, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "Field property of a Message", // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + 0, // tp_methods + 0, // tp_members + field::Getters, // tp_getset + 0, // tp_base + 0, // tp_dict + (descrgetfunc)field::DescrGet, // tp_descr_get + (descrsetfunc)field::DescrSet, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init + 0, // tp_alloc + 0, // tp_new +}; +PyTypeObject* CFieldProperty_Type = &_CFieldProperty_Type; + +PyObject* NewFieldProperty(const FieldDescriptor* field_descriptor) { + // Create a new descriptor object + PyMessageFieldProperty* property = + PyObject_New(PyMessageFieldProperty, CFieldProperty_Type); + if (property == NULL) { + return NULL; + } + property->field_descriptor = field_descriptor; + return reinterpret_cast<PyObject*>(property); +} + +} // namespace python +} // namespace protobuf +} // namespace google diff --git a/python/google/protobuf/pyext/field.h b/python/google/protobuf/pyext/field.h new file mode 100755 index 00000000..7b4660ca --- /dev/null +++ b/python/google/protobuf/pyext/field.h @@ -0,0 +1,59 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_FIELD_H__ +#define GOOGLE_PROTOBUF_PYTHON_CPP_FIELD_H__ + +#include <Python.h> + +namespace google { +namespace protobuf { + +class FieldDescriptor; + +namespace python { + +// A data descriptor that represents a field in a Message class. +struct PyMessageFieldProperty { + PyObject_HEAD; + + // This pointer is owned by the same pool as the Message class it belongs to. + const FieldDescriptor* field_descriptor; +}; + +extern PyTypeObject* CFieldProperty_Type; + +PyObject* NewFieldProperty(const FieldDescriptor* field_descriptor); + +} // namespace python +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_PYTHON_CPP_FIELD_H__ diff --git a/python/google/protobuf/pyext/map_container.cc b/python/google/protobuf/pyext/map_container.cc index 6d7ee285..77c61706 100644 --- a/python/google/protobuf/pyext/map_container.cc +++ b/python/google/protobuf/pyext/map_container.cc @@ -68,6 +68,8 @@ class MapReflectionFriend { static PyObject* MessageMapGetItem(PyObject* _self, PyObject* key); static int ScalarMapSetItem(PyObject* _self, PyObject* key, PyObject* v); static int MessageMapSetItem(PyObject* _self, PyObject* key, PyObject* v); + static PyObject* ScalarMapToStr(PyObject* _self); + static PyObject* MessageMapToStr(PyObject* _self); }; struct MapIterator { @@ -199,26 +201,26 @@ static PyObject* MapKeyToPython(const FieldDescriptor* field_descriptor, // This is only used for ScalarMap, so we don't need to handle the // CPPTYPE_MESSAGE case. PyObject* MapValueRefToPython(const FieldDescriptor* field_descriptor, - MapValueRef* value) { + const MapValueRef& value) { switch (field_descriptor->cpp_type()) { case FieldDescriptor::CPPTYPE_INT32: - return PyInt_FromLong(value->GetInt32Value()); + return PyInt_FromLong(value.GetInt32Value()); case FieldDescriptor::CPPTYPE_INT64: - return PyLong_FromLongLong(value->GetInt64Value()); + return PyLong_FromLongLong(value.GetInt64Value()); case FieldDescriptor::CPPTYPE_UINT32: - return PyInt_FromSize_t(value->GetUInt32Value()); + return PyInt_FromSize_t(value.GetUInt32Value()); case FieldDescriptor::CPPTYPE_UINT64: - return PyLong_FromUnsignedLongLong(value->GetUInt64Value()); + return PyLong_FromUnsignedLongLong(value.GetUInt64Value()); case FieldDescriptor::CPPTYPE_FLOAT: - return PyFloat_FromDouble(value->GetFloatValue()); + return PyFloat_FromDouble(value.GetFloatValue()); case FieldDescriptor::CPPTYPE_DOUBLE: - return PyFloat_FromDouble(value->GetDoubleValue()); + return PyFloat_FromDouble(value.GetDoubleValue()); case FieldDescriptor::CPPTYPE_BOOL: - return PyBool_FromLong(value->GetBoolValue()); + return PyBool_FromLong(value.GetBoolValue()); case FieldDescriptor::CPPTYPE_STRING: - return ToStringObject(field_descriptor, value->GetStringValue()); + return ToStringObject(field_descriptor, value.GetStringValue()); case FieldDescriptor::CPPTYPE_ENUM: - return PyInt_FromLong(value->GetEnumValue()); + return PyInt_FromLong(value.GetEnumValue()); default: PyErr_Format( PyExc_SystemError, "Couldn't convert type %d to value", @@ -472,7 +474,7 @@ PyObject* MapReflectionFriend::ScalarMapGetItem(PyObject* _self, self->version++; } - return MapValueRefToPython(self->value_field_descriptor, &value); + return MapValueRefToPython(self->value_field_descriptor, value); } int MapReflectionFriend::ScalarMapSetItem(PyObject* _self, PyObject* key, @@ -535,10 +537,47 @@ static PyObject* ScalarMapGet(PyObject* self, PyObject* args) { } } +PyObject* MapReflectionFriend::ScalarMapToStr(PyObject* _self) { + ScopedPyObjectPtr dict(PyDict_New()); + if (dict == NULL) { + return NULL; + } + ScopedPyObjectPtr key; + ScopedPyObjectPtr value; + + MapContainer* self = GetMap(_self); + Message* message = self->GetMutableMessage(); + const Reflection* reflection = message->GetReflection(); + for (google::protobuf::MapIterator it = reflection->MapBegin( + message, self->parent_field_descriptor); + it != reflection->MapEnd(message, self->parent_field_descriptor); + ++it) { + key.reset(MapKeyToPython(self->key_field_descriptor, + it.GetKey())); + if (key == NULL) { + return NULL; + } + value.reset(MapValueRefToPython(self->value_field_descriptor, + it.GetValueRef())); + if (value == NULL) { + return NULL; + } + if (PyDict_SetItem(dict.get(), key.get(), value.get()) < 0) { + return NULL; + } + } + return PyObject_Repr(dict.get()); +} + static void ScalarMapDealloc(PyObject* _self) { MapContainer* self = GetMap(_self); self->owner.reset(); - Py_TYPE(_self)->tp_free(_self); + PyTypeObject *type = Py_TYPE(_self); + type->tp_free(_self); + if (type->tp_flags & Py_TPFLAGS_HEAPTYPE) { + // With Python3, the Map class is not static, and must be managed. + Py_DECREF(type); + } } static PyMethodDef ScalarMapMethods[] = { @@ -570,6 +609,7 @@ PyTypeObject *ScalarMapContainer_Type; {Py_mp_ass_subscript, (void *)MapReflectionFriend::ScalarMapSetItem}, {Py_tp_methods, (void *)ScalarMapMethods}, {Py_tp_iter, (void *)MapReflectionFriend::GetIterator}, + {Py_tp_repr, (void *)MapReflectionFriend::ScalarMapToStr}, {0, 0}, }; @@ -597,7 +637,7 @@ PyTypeObject *ScalarMapContainer_Type; 0, // tp_getattr 0, // tp_setattr 0, // tp_compare - 0, // tp_repr + MapReflectionFriend::ScalarMapToStr, // tp_repr 0, // tp_as_number 0, // tp_as_sequence &ScalarMapMappingMethods, // tp_as_mapping @@ -634,7 +674,8 @@ static MessageMapContainer* GetMessageMap(PyObject* obj) { return reinterpret_cast<MessageMapContainer*>(obj); } -static PyObject* GetCMessage(MessageMapContainer* self, Message* message) { +static PyObject* GetCMessage(MessageMapContainer* self, Message* message, + bool insert_message_dict) { // Get or create the CMessage object corresponding to this message. ScopedPyObjectPtr key(PyLong_FromVoidPtr(message)); PyObject* ret = PyDict_GetItem(self->message_dict, key.get()); @@ -649,10 +690,11 @@ static PyObject* GetCMessage(MessageMapContainer* self, Message* message) { cmsg->owner = self->owner; cmsg->message = message; cmsg->parent = self->parent; - - if (PyDict_SetItem(self->message_dict, key.get(), ret) < 0) { - Py_DECREF(ret); - return NULL; + if (insert_message_dict) { + if (PyDict_SetItem(self->message_dict, key.get(), ret) < 0) { + Py_DECREF(ret); + return NULL; + } } } else { Py_INCREF(ret); @@ -781,7 +823,41 @@ PyObject* MapReflectionFriend::MessageMapGetItem(PyObject* _self, self->version++; } - return GetCMessage(self, value.MutableMessageValue()); + return GetCMessage(self, value.MutableMessageValue(), true); +} + +PyObject* MapReflectionFriend::MessageMapToStr(PyObject* _self) { + ScopedPyObjectPtr dict(PyDict_New()); + if (dict == NULL) { + return NULL; + } + ScopedPyObjectPtr key; + ScopedPyObjectPtr value; + + MessageMapContainer* self = GetMessageMap(_self); + Message* message = self->GetMutableMessage(); + const Reflection* reflection = message->GetReflection(); + for (google::protobuf::MapIterator it = reflection->MapBegin( + message, self->parent_field_descriptor); + it != reflection->MapEnd(message, self->parent_field_descriptor); + ++it) { + key.reset(MapKeyToPython(self->key_field_descriptor, + it.GetKey())); + if (key == NULL) { + return NULL; + } + // Do not insert the cmessage to self->message_dict because + // the returned CMessage will not escape this function. + value.reset(GetCMessage( + self, it.MutableValueRef()->MutableMessageValue(), false)); + if (value == NULL) { + return NULL; + } + if (PyDict_SetItem(dict.get(), key.get(), value.get()) < 0) { + return NULL; + } + } + return PyObject_Repr(dict.get()); } PyObject* MessageMapGet(PyObject* self, PyObject* args) { @@ -813,7 +889,12 @@ static void MessageMapDealloc(PyObject* _self) { self->owner.reset(); Py_DECREF(self->message_dict); Py_DECREF(self->message_class); - Py_TYPE(_self)->tp_free(_self); + PyTypeObject *type = Py_TYPE(_self); + type->tp_free(_self); + if (type->tp_flags & Py_TPFLAGS_HEAPTYPE) { + // With Python3, the Map class is not static, and must be managed. + Py_DECREF(type); + } } static PyMethodDef MessageMapMethods[] = { @@ -847,6 +928,7 @@ PyTypeObject *MessageMapContainer_Type; {Py_mp_ass_subscript, (void *)MapReflectionFriend::MessageMapSetItem}, {Py_tp_methods, (void *)MessageMapMethods}, {Py_tp_iter, (void *)MapReflectionFriend::GetIterator}, + {Py_tp_repr, (void *)MapReflectionFriend::MessageMapToStr}, {0, 0} }; @@ -874,7 +956,7 @@ PyTypeObject *MessageMapContainer_Type; 0, // tp_getattr 0, // tp_setattr 0, // tp_compare - 0, // tp_repr + MapReflectionFriend::MessageMapToStr, // tp_repr 0, // tp_as_number 0, // tp_as_sequence &MessageMapMappingMethods, // tp_as_mapping @@ -1027,17 +1109,15 @@ bool InitMapContainers() { return false; } - if (!PyObject_TypeCheck(mutable_mapping.get(), &PyType_Type)) { - return false; - } - Py_INCREF(mutable_mapping.get()); #if PY_MAJOR_VERSION >= 3 - PyObject* bases = PyTuple_New(1); - PyTuple_SET_ITEM(bases, 0, mutable_mapping.get()); + ScopedPyObjectPtr bases(PyTuple_Pack(1, mutable_mapping.get())); + if (bases == NULL) { + return false; + } ScalarMapContainer_Type = reinterpret_cast<PyTypeObject*>( - PyType_FromSpecWithBases(&ScalarMapContainer_Type_spec, bases)); + PyType_FromSpecWithBases(&ScalarMapContainer_Type_spec, bases.get())); #else _ScalarMapContainer_Type.tp_base = reinterpret_cast<PyTypeObject*>(mutable_mapping.get()); @@ -1055,7 +1135,7 @@ bool InitMapContainers() { #if PY_MAJOR_VERSION >= 3 MessageMapContainer_Type = reinterpret_cast<PyTypeObject*>( - PyType_FromSpecWithBases(&MessageMapContainer_Type_spec, bases)); + PyType_FromSpecWithBases(&MessageMapContainer_Type_spec, bases.get())); #else Py_INCREF(mutable_mapping.get()); _MessageMapContainer_Type.tp_base = diff --git a/python/google/protobuf/pyext/map_container.h b/python/google/protobuf/pyext/map_container.h index 111fafbf..7e77b027 100644 --- a/python/google/protobuf/pyext/map_container.h +++ b/python/google/protobuf/pyext/map_container.h @@ -120,6 +120,6 @@ extern PyObject* NewMessageMapContainer( } // namespace python } // namespace protobuf - } // namespace google + #endif // GOOGLE_PROTOBUF_PYTHON_CPP_MAP_CONTAINER_H__ diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc index 53736b9c..fecb9364 100644 --- a/python/google/protobuf/pyext/message.cc +++ b/python/google/protobuf/pyext/message.cc @@ -45,12 +45,11 @@ #ifndef Py_TYPE #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) #endif -#include <google/protobuf/descriptor.pb.h> #include <google/protobuf/stubs/common.h> #include <google/protobuf/stubs/logging.h> #include <google/protobuf/io/coded_stream.h> #include <google/protobuf/io/zero_copy_stream_impl_lite.h> -#include <google/protobuf/util/message_differencer.h> +#include <google/protobuf/descriptor.pb.h> #include <google/protobuf/descriptor.h> #include <google/protobuf/message.h> #include <google/protobuf/text_format.h> @@ -58,12 +57,18 @@ #include <google/protobuf/pyext/descriptor.h> #include <google/protobuf/pyext/descriptor_pool.h> #include <google/protobuf/pyext/extension_dict.h> -#include <google/protobuf/pyext/repeated_composite_container.h> -#include <google/protobuf/pyext/repeated_scalar_container.h> +#include <google/protobuf/pyext/field.h> #include <google/protobuf/pyext/map_container.h> #include <google/protobuf/pyext/message_factory.h> +#include <google/protobuf/pyext/repeated_composite_container.h> +#include <google/protobuf/pyext/repeated_scalar_container.h> +#include <google/protobuf/pyext/unknown_fields.h> #include <google/protobuf/pyext/safe_numerics.h> #include <google/protobuf/pyext/scoped_pyobject_ptr.h> +#include <google/protobuf/util/message_differencer.h> +#include <google/protobuf/stubs/strutil.h> + +#include <google/protobuf/port_def.inc> #if PY_MAJOR_VERSION >= 3 #define PyInt_AsLong PyLong_AsLong @@ -72,16 +77,19 @@ #define PyString_Check PyUnicode_Check #define PyString_FromString PyUnicode_FromString #define PyString_FromStringAndSize PyUnicode_FromStringAndSize + #define PyString_FromFormat PyUnicode_FromFormat #if PY_VERSION_HEX < 0x03030000 #error "Python 3.0 - 3.2 are not supported." #else #define PyString_AsString(ob) \ (PyUnicode_Check(ob)? PyUnicode_AsUTF8(ob): PyBytes_AsString(ob)) - #define PyString_AsStringAndSize(ob, charpp, sizep) \ - (PyUnicode_Check(ob)? \ - ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \ - PyBytes_AsStringAndSize(ob, (charpp), (sizep))) - #endif +#define PyString_AsStringAndSize(ob, charpp, sizep) \ + (PyUnicode_Check(ob) ? ((*(charpp) = const_cast<char*>( \ + PyUnicode_AsUTF8AndSize(ob, (sizep)))) == NULL \ + ? -1 \ + : 0) \ + : PyBytes_AsStringAndSize(ob, (charpp), (sizep))) +#endif #endif namespace google { @@ -99,44 +107,27 @@ namespace message_meta { static int InsertEmptyWeakref(PyTypeObject* base); namespace { -// Copied oveer from internal 'google/protobuf/stubs/strutil.h'. -inline void UpperString(string * s) { +// Copied over from internal 'google/protobuf/stubs/strutil.h'. +inline void LowerString(string * s) { string::iterator end = s->end(); for (string::iterator i = s->begin(); i != end; ++i) { - // toupper() changes based on locale. We don't want this! - if ('a' <= *i && *i <= 'z') *i += 'A' - 'a'; + // tolower() changes based on locale. We don't want this! + if ('A' <= *i && *i <= 'Z') *i += 'a' - 'A'; } } } -// Add the number of a field descriptor to the containing message class. -// Equivalent to: -// _cls.<field>_FIELD_NUMBER = <number> -static bool AddFieldNumberToClass( - PyObject* cls, const FieldDescriptor* field_descriptor) { - string constant_name = field_descriptor->name() + "_FIELD_NUMBER"; - UpperString(&constant_name); - ScopedPyObjectPtr attr_name(PyString_FromStringAndSize( - constant_name.c_str(), constant_name.size())); - if (attr_name == NULL) { - return false; - } - ScopedPyObjectPtr number(PyInt_FromLong(field_descriptor->number())); - if (number == NULL) { - return false; - } - if (PyObject_SetAttr(cls, attr_name.get(), number.get()) == -1) { - return false; - } - return true; -} - - // Finalize the creation of the Message class. static int AddDescriptors(PyObject* cls, const Descriptor* descriptor) { // For each field set: cls.<field>_FIELD_NUMBER = <number> for (int i = 0; i < descriptor->field_count(); ++i) { - if (!AddFieldNumberToClass(cls, descriptor->field(i))) { + const FieldDescriptor* field_descriptor = descriptor->field(i); + ScopedPyObjectPtr property(NewFieldProperty(field_descriptor)); + if (property == NULL) { + return -1; + } + if (PyObject_SetAttrString(cls, field_descriptor->name().c_str(), + property.get()) < 0) { return -1; } } @@ -193,11 +184,6 @@ static int AddDescriptors(PyObject* cls, const Descriptor* descriptor) { cls, field->name().c_str(), extension_field.get()) == -1) { return -1; } - - // For each extension set cls.<extension name>_FIELD_NUMBER = <number>. - if (!AddFieldNumberToClass(cls, field)) { - return -1; - } } return 0; @@ -265,10 +251,10 @@ static PyObject* New(PyTypeObject* type, PyObject* well_known_class = PyDict_GetItemString( WKT_classes, message_descriptor->full_name().c_str()); if (well_known_class == NULL) { - new_args.reset(Py_BuildValue("s(OO)O", name, &CMessage_Type, + new_args.reset(Py_BuildValue("s(OO)O", name, CMessage_Type, PythonMessage_class, dict)); } else { - new_args.reset(Py_BuildValue("s(OOO)O", name, &CMessage_Type, + new_args.reset(Py_BuildValue("s(OOO)O", name, CMessage_Type, PythonMessage_class, well_known_class, dict)); } @@ -285,7 +271,7 @@ static PyObject* New(PyTypeObject* type, // Insert the empty weakref into the base classes. if (InsertEmptyWeakref( reinterpret_cast<PyTypeObject*>(PythonMessage_class)) < 0 || - InsertEmptyWeakref(&CMessage_Type) < 0) { + InsertEmptyWeakref(CMessage_Type) < 0) { return NULL; } @@ -353,6 +339,13 @@ static int InsertEmptyWeakref(PyTypeObject *base_type) { // The _extensions_by_name dictionary is built on every access. // TODO(amauryfa): Migrate all users to pool.FindAllExtensions() static PyObject* GetExtensionsByName(CMessageClass *self, void *closure) { + if (self->message_descriptor == NULL) { + // This is the base Message object, simply raise AttributeError. + PyErr_SetString(PyExc_AttributeError, + "Base Message class has no DESCRIPTOR"); + return NULL; + } + const PyDescriptorPool* pool = self->py_message_factory->pool; std::vector<const FieldDescriptor*> extensions; @@ -376,6 +369,13 @@ static PyObject* GetExtensionsByName(CMessageClass *self, void *closure) { // The _extensions_by_number dictionary is built on every access. // TODO(amauryfa): Migrate all users to pool.FindExtensionByNumber() static PyObject* GetExtensionsByNumber(CMessageClass *self, void *closure) { + if (self->message_descriptor == NULL) { + // This is the base Message object, simply raise AttributeError. + PyErr_SetString(PyExc_AttributeError, + "Base Message class has no DESCRIPTOR"); + return NULL; + } + const PyDescriptorPool* pool = self->py_message_factory->pool; std::vector<const FieldDescriptor*> extensions; @@ -405,9 +405,51 @@ static PyGetSetDef Getters[] = { {NULL} }; +// Compute some class attributes on the fly: +// - All the _FIELD_NUMBER attributes, for all fields and nested extensions. +// Returns a new reference, or NULL with an exception set. +static PyObject* GetClassAttribute(CMessageClass *self, PyObject* name) { + char* attr; + Py_ssize_t attr_size; + static const char kSuffix[] = "_FIELD_NUMBER"; + if (PyString_AsStringAndSize(name, &attr, &attr_size) >= 0 && + strings::EndsWith(StringPiece(attr, attr_size), kSuffix)) { + string field_name(attr, attr_size - sizeof(kSuffix) + 1); + LowerString(&field_name); + + // Try to find a field with the given name, without the suffix. + const FieldDescriptor* field = + self->message_descriptor->FindFieldByLowercaseName(field_name); + if (!field) { + // Search nested extensions as well. + field = + self->message_descriptor->FindExtensionByLowercaseName(field_name); + } + if (field) { + return PyInt_FromLong(field->number()); + } + } + PyErr_SetObject(PyExc_AttributeError, name); + return NULL; +} + +static PyObject* GetAttr(CMessageClass* self, PyObject* name) { + PyObject* result = CMessageClass_Type->tp_base->tp_getattro( + reinterpret_cast<PyObject*>(self), name); + if (result != NULL) { + return result; + } + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + return NULL; + } + + PyErr_Clear(); + return GetClassAttribute(self, name); +} + } // namespace message_meta -PyTypeObject CMessageClass_Type = { +static PyTypeObject _CMessageClass_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME ".MessageMeta", // tp_name sizeof(CMessageClass), // tp_basicsize @@ -424,7 +466,7 @@ PyTypeObject CMessageClass_Type = { 0, // tp_hash 0, // tp_call 0, // tp_str - 0, // tp_getattro + (getattrofunc)message_meta::GetAttr, // tp_getattro 0, // tp_setattro 0, // tp_as_buffer Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, // tp_flags @@ -447,9 +489,10 @@ PyTypeObject CMessageClass_Type = { 0, // tp_alloc message_meta::New, // tp_new }; +PyTypeObject* CMessageClass_Type = &_CMessageClass_Type; static CMessageClass* CheckMessageClass(PyTypeObject* cls) { - if (!PyObject_TypeCheck(cls, &CMessageClass_Type)) { + if (!PyObject_TypeCheck(cls, CMessageClass_Type)) { PyErr_Format(PyExc_TypeError, "Class %s is not a Message", cls->tp_name); return NULL; } @@ -487,10 +530,20 @@ struct ChildVisitor { } // Returns 0 on success, -1 on failure. + int VisitMapContainer(MapContainer* container) { + return 0; + } + + // Returns 0 on success, -1 on failure. int VisitCMessage(CMessage* cmessage, const FieldDescriptor* field_descriptor) { return 0; } + + // Returns 0 on success, -1 on failure. + int VisitUnknownFieldSet(PyUnknownFields* unknown_field_set) { + return 0; + } }; // Apply a function to a composite field. Does nothing if child is of @@ -538,34 +591,19 @@ int ForEachCompositeField(CMessage* self, Visitor visitor) { // Visit normal fields. if (self->composite_fields) { - // Never use self->message in this function, it may be already freed. - const Descriptor* message_descriptor = - GetMessageDescriptor(Py_TYPE(self)); - while (PyDict_Next(self->composite_fields, &pos, &key, &field)) { - Py_ssize_t key_str_size; - char *key_str_data; - if (PyString_AsStringAndSize(key, &key_str_data, &key_str_size) != 0) - return -1; - const string key_str(key_str_data, key_str_size); - const FieldDescriptor* descriptor = - message_descriptor->FindFieldByName(key_str); - if (descriptor != NULL) { - if (VisitCompositeField(descriptor, field, visitor) == -1) - return -1; - } + for (CMessage::CompositeFieldsMap::iterator it = + self->composite_fields->begin(); + it != self->composite_fields->end(); it++) { + const FieldDescriptor* descriptor = it->first; + PyObject* field = it->second; + if (VisitCompositeField(descriptor, field, visitor) == -1) return -1; } } - // Visit extension fields. - if (self->extensions != NULL) { - pos = 0; - while (PyDict_Next(self->extensions->values, &pos, &key, &field)) { - const FieldDescriptor* descriptor = cmessage::GetExtensionDescriptor(key); - if (descriptor == NULL) - return -1; - if (VisitCompositeField(descriptor, field, visitor) == -1) - return -1; - } + if (self->unknown_field_set) { + PyUnknownFields* unknown_field_set = + reinterpret_cast<PyUnknownFields*>(self->unknown_field_set); + visitor.VisitUnknownFieldSet(unknown_field_set); } return 0; @@ -577,8 +615,12 @@ PyObject* EncodeError_class; PyObject* DecodeError_class; PyObject* PickleError_class; -/* Is 64bit */ +// Format an error message for unexpected types. +// Always return with an exception set. void FormatTypeError(PyObject* arg, char* expected_types) { + // This function is often called with an exception set. + // Clear it to call PyObject_Repr() in good conditions. + PyErr_Clear(); PyObject* repr = PyObject_Repr(arg); if (repr) { PyErr_Format(PyExc_TypeError, @@ -602,7 +644,7 @@ void OutOfRangeError(PyObject* arg) { template<class RangeType, class ValueType> bool VerifyIntegerCastAndRange(PyObject* arg, ValueType value) { - if (GOOGLE_PREDICT_FALSE(value == -1 && PyErr_Occurred())) { + if (PROTOBUF_PREDICT_FALSE(value == -1 && PyErr_Occurred())) { if (PyErr_ExceptionMatches(PyExc_OverflowError)) { // Replace it with the same ValueError as pure python protos instead of // the default one. @@ -611,7 +653,7 @@ bool VerifyIntegerCastAndRange(PyObject* arg, ValueType value) { } // Otherwise propagate existing error. return false; } - if (GOOGLE_PREDICT_FALSE(!IsValidNumericCast<RangeType>(value))) { + if (PROTOBUF_PREDICT_FALSE(!IsValidNumericCast<RangeType>(value))) { OutOfRangeError(arg); return false; } @@ -623,22 +665,22 @@ bool CheckAndGetInteger(PyObject* arg, T* value) { // The fast path. #if PY_MAJOR_VERSION < 3 // For the typical case, offer a fast path. - if (GOOGLE_PREDICT_TRUE(PyInt_Check(arg))) { + if (PROTOBUF_PREDICT_TRUE(PyInt_Check(arg))) { long int_result = PyInt_AsLong(arg); - if (GOOGLE_PREDICT_TRUE(IsValidNumericCast<T>(int_result))) { + if (PROTOBUF_PREDICT_TRUE(IsValidNumericCast<T>(int_result))) { *value = static_cast<T>(int_result); return true; } else { OutOfRangeError(arg); return false; } - } + } #endif // This effectively defines an integer as "an object that can be cast as // an integer and can be used as an ordinal number". // This definition includes everything that implements numbers.Integral // and shouldn't cast the net too wide. - if (GOOGLE_PREDICT_FALSE(!PyIndex_Check(arg))) { + if (PROTOBUF_PREDICT_FALSE(!PyIndex_Check(arg))) { FormatTypeError(arg, "int, long"); return false; } @@ -655,7 +697,7 @@ bool CheckAndGetInteger(PyObject* arg, T* value) { // Unlike PyLong_AsLongLong, PyLong_AsUnsignedLongLong is very // picky about the exact type. PyObject* casted = PyNumber_Long(arg); - if (GOOGLE_PREDICT_FALSE(casted == nullptr)) { + if (PROTOBUF_PREDICT_FALSE(casted == nullptr)) { // Propagate existing error. return false; } @@ -680,7 +722,7 @@ bool CheckAndGetInteger(PyObject* arg, T* value) { // Valid subclasses of numbers.Integral should have a __long__() method // so fall back to that. PyObject* casted = PyNumber_Long(arg); - if (GOOGLE_PREDICT_FALSE(casted == nullptr)) { + if (PROTOBUF_PREDICT_FALSE(casted == nullptr)) { // Propagate existing error. return false; } @@ -706,7 +748,7 @@ template bool CheckAndGetInteger<uint64>(PyObject*, uint64*); bool CheckAndGetDouble(PyObject* arg, double* value) { *value = PyFloat_AsDouble(arg); - if (GOOGLE_PREDICT_FALSE(*value == -1 && PyErr_Occurred())) { + if (PROTOBUF_PREDICT_FALSE(*value == -1 && PyErr_Occurred())) { FormatTypeError(arg, "int, long, float"); return false; } @@ -859,7 +901,7 @@ bool CheckFieldBelongsToMessage(const FieldDescriptor* field_descriptor, namespace cmessage { PyMessageFactory* GetFactoryForMessage(CMessage* message) { - GOOGLE_DCHECK(PyObject_TypeCheck(message, &CMessage_Type)); + GOOGLE_DCHECK(PyObject_TypeCheck(message, CMessage_Type)); return reinterpret_cast<CMessageClass*>(Py_TYPE(message))->py_message_factory; } @@ -883,22 +925,20 @@ static int MaybeReleaseOverlappingOneofField( // Non-message fields don't need to be released. return 0; } - const char* field_name = existing_field->name().c_str(); - PyObject* child_message = cmessage->composite_fields ? - PyDict_GetItemString(cmessage->composite_fields, field_name) : NULL; - if (child_message == NULL) { - // No python reference to this field so no need to release. - return 0; - } - - if (InternalReleaseFieldByDescriptor( - cmessage, existing_field, child_message) < 0) { - return -1; + if (cmessage->composite_fields) { + CMessage::CompositeFieldsMap::iterator iterator = + cmessage->composite_fields->find(existing_field); + if (iterator != cmessage->composite_fields->end()) { + if (InternalReleaseFieldByDescriptor(cmessage, existing_field, + iterator->second) < 0) { + return -1; + } + Py_DECREF(iterator->second); + cmessage->composite_fields->erase(iterator); + } } - return PyDict_DelItemString(cmessage->composite_fields, field_name); -#else - return 0; #endif + return 0; } // --------------------------------------------------------------------- @@ -937,10 +977,49 @@ struct FixupMessageReference : public ChildVisitor { return 0; } + int VisitUnknownFieldSet(PyUnknownFields* unknown_field_set) { + const Reflection* reflection = message_->GetReflection(); + unknown_field_set->fields = &reflection->GetUnknownFields(*message_); + return 0; + } + private: Message* message_; }; +// After a Merge, visit every sub-message that was read-only, and +// eventually update their pointer if the Merge operation modified them. +struct FixupMessageAfterMerge : public FixupMessageReference { + explicit FixupMessageAfterMerge(CMessage* parent) : + FixupMessageReference(parent->message), + parent_cmessage(parent), message(parent->message) {} + + int VisitCMessage(CMessage* cmessage, + const FieldDescriptor* field_descriptor) { + if (cmessage->read_only == false) { + return 0; + } + if (message->GetReflection()->HasField(*message, field_descriptor)) { + Message* mutable_message = GetMutableMessage( + parent_cmessage, field_descriptor); + if (mutable_message == NULL) { + return -1; + } + cmessage->message = mutable_message; + cmessage->read_only = false; + if (ForEachCompositeField( + cmessage, FixupMessageAfterMerge(cmessage)) == -1) { + return -1; + } + } + return 0; + } + + private: + CMessage* parent_cmessage; + Message* message; +}; + int AssureWritable(CMessage* self) { if (self == NULL || !self->read_only) { return 0; @@ -974,10 +1053,8 @@ int AssureWritable(CMessage* self) { // When a CMessage is made writable its Message pointer is updated // to point to a new mutable Message. When that happens we need to // update any references to the old, read-only CMessage. There are - // four places such references occur: RepeatedScalarContainer, - // RepeatedCompositeContainer, MapContainer, and ExtensionDict. - if (self->extensions != NULL) - self->extensions->message = self->message; + // three places such references occur: RepeatedScalarContainer, + // RepeatedCompositeContainer, and MapContainer. if (ForEachCompositeField(self, FixupMessageReference(self->message)) == -1) return -1; @@ -986,27 +1063,6 @@ int AssureWritable(CMessage* self) { // --- Globals: -// Retrieve a C++ FieldDescriptor for a message attribute. -// The C++ message must be valid. -// TODO(amauryfa): This function should stay internal, because exception -// handling is not consistent. -static const FieldDescriptor* GetFieldDescriptor( - CMessage* self, PyObject* name) { - const Descriptor *message_descriptor = self->message->GetDescriptor(); - char* field_name; - Py_ssize_t size; - if (PyString_AsStringAndSize(name, &field_name, &size) < 0) { - return NULL; - } - const FieldDescriptor *field_descriptor = - message_descriptor->FindFieldByName(string(field_name, size)); - if (field_descriptor == NULL) { - // Note: No exception is set! - return NULL; - } - return field_descriptor; -} - // Retrieve a C++ FieldDescriptor for an extension handle. const FieldDescriptor* GetExtensionDescriptor(PyObject* extension) { ScopedPyObjectPtr cdescriptor; @@ -1038,7 +1094,7 @@ static PyObject* GetIntegerEnumValue(const FieldDescriptor& descriptor, const EnumValueDescriptor* enum_value_descriptor = enum_descriptor->FindValueByName(string(enum_label, size)); if (enum_value_descriptor == NULL) { - PyErr_SetString(PyExc_ValueError, "unknown enum label"); + PyErr_Format(PyExc_ValueError, "unknown enum label \"%s\"", enum_label); return NULL; } return PyInt_FromLong(enum_value_descriptor->number()); @@ -1052,11 +1108,10 @@ static PyObject* GetIntegerEnumValue(const FieldDescriptor& descriptor, // needs to do this to make sure CMessages stay alive if they're still // referenced after deletion. Repeated scalar container doesn't need to worry. int InternalDeleteRepeatedField( - CMessage* self, + Message* message, const FieldDescriptor* field_descriptor, PyObject* slice, PyObject* cmessage_list) { - Message* message = self->message; Py_ssize_t length, from, to, step, slice_length; const Reflection* reflection = message->GetReflection(); int min, max; @@ -1134,7 +1189,7 @@ int InternalDeleteRepeatedField( CMessage* last_cmessage = reinterpret_cast<CMessage*>( PyList_GET_ITEM(cmessage_list, PyList_GET_SIZE(cmessage_list) - 1)); repeated_composite_container::ReleaseLastTo( - self, field_descriptor, last_cmessage); + message, field_descriptor, last_cmessage); if (PySequence_DelItem(cmessage_list, -1) < 0) { return -1; } @@ -1160,23 +1215,28 @@ int InitAttributes(CMessage* self, PyObject* args, PyObject* kwargs) { PyObject* name; PyObject* value; while (PyDict_Next(kwargs, &pos, &name, &value)) { - if (!PyString_Check(name)) { + if (!(PyString_Check(name) || PyUnicode_Check(name))) { PyErr_SetString(PyExc_ValueError, "Field name must be a string"); return -1; } - const FieldDescriptor* descriptor = GetFieldDescriptor(self, name); - if (descriptor == NULL) { + ScopedPyObjectPtr property( + PyObject_GetAttr(reinterpret_cast<PyObject*>(Py_TYPE(self)), name)); + if (property == NULL || + !PyObject_TypeCheck(property.get(), CFieldProperty_Type)) { PyErr_Format(PyExc_ValueError, "Protocol message %s has no \"%s\" field.", self->message->GetDescriptor()->name().c_str(), PyString_AsString(name)); return -1; } + const FieldDescriptor* descriptor = + reinterpret_cast<PyMessageFieldProperty*>(property.get()) + ->field_descriptor; if (value == Py_None) { // field=None is the same as no field at all. continue; } if (descriptor->is_map()) { - ScopedPyObjectPtr map(GetAttr(reinterpret_cast<PyObject*>(self), name)); + ScopedPyObjectPtr map(GetFieldValue(self, descriptor)); const FieldDescriptor* value_descriptor = descriptor->message_type()->FindFieldByName("value"); if (value_descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { @@ -1204,8 +1264,7 @@ int InitAttributes(CMessage* self, PyObject* args, PyObject* kwargs) { } } } else if (descriptor->label() == FieldDescriptor::LABEL_REPEATED) { - ScopedPyObjectPtr container( - GetAttr(reinterpret_cast<PyObject*>(self), name)); + ScopedPyObjectPtr container(GetFieldValue(self, descriptor)); if (container == NULL) { return -1; } @@ -1272,8 +1331,7 @@ int InitAttributes(CMessage* self, PyObject* args, PyObject* kwargs) { } } } else if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { - ScopedPyObjectPtr message( - GetAttr(reinterpret_cast<PyObject*>(self), name)); + ScopedPyObjectPtr message(GetFieldValue(self, descriptor)); if (message == NULL) { return -1; } @@ -1297,9 +1355,9 @@ int InitAttributes(CMessage* self, PyObject* args, PyObject* kwargs) { if (new_val == NULL) { return -1; } + value = new_val.get(); } - if (SetAttr(reinterpret_cast<PyObject*>(self), name, - (new_val.get() == NULL) ? value : new_val.get()) < 0) { + if (SetFieldValue(self, descriptor, value) < 0) { return -1; } } @@ -1322,10 +1380,11 @@ CMessage* NewEmptyMessage(CMessageClass* type) { self->parent = NULL; self->parent_field_descriptor = NULL; self->read_only = false; - self->extensions = NULL; self->composite_fields = NULL; + self->unknown_field_set = NULL; + return self; } @@ -1408,12 +1467,20 @@ static void Dealloc(CMessage* self) { } // Null out all weak references from children to this message. GOOGLE_CHECK_EQ(0, ForEachCompositeField(self, ClearWeakReferences())); - if (self->extensions) { - self->extensions->parent = NULL; - } - Py_CLEAR(self->extensions); - Py_CLEAR(self->composite_fields); + if (self->composite_fields) { + for (CMessage::CompositeFieldsMap::iterator it = + self->composite_fields->begin(); + it != self->composite_fields->end(); it++) { + Py_DECREF(it->second); + } + delete self->composite_fields; + } + if (self->unknown_field_set) { + unknown_fields::Clear( + reinterpret_cast<PyUnknownFields*>(self->unknown_field_set)); + Py_CLEAR(self->unknown_field_set); + } self->owner.~ThreadUnsafeSharedPtr<Message>(); Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self)); } @@ -1529,7 +1596,7 @@ PyObject* HasField(CMessage* self, PyObject* arg) { return NULL; } #else - field_name = PyUnicode_AsUTF8AndSize(arg, &size); + field_name = const_cast<char*>(PyUnicode_AsUTF8AndSize(arg, &size)); if (!field_name) { return NULL; } @@ -1564,13 +1631,16 @@ PyObject* ClearExtension(CMessage* self, PyObject* extension) { if (descriptor == NULL) { return NULL; } - if (self->extensions != NULL) { - PyObject* value = PyDict_GetItem(self->extensions->values, extension); - if (value != NULL) { - if (InternalReleaseFieldByDescriptor(self, descriptor, value) < 0) { + if (self->composite_fields != NULL) { + CMessage::CompositeFieldsMap::iterator iterator = + self->composite_fields->find(descriptor); + if (iterator != self->composite_fields->end()) { + if (InternalReleaseFieldByDescriptor(self, descriptor, + iterator->second) < 0) { return NULL; } - PyDict_DelItem(self->extensions->values, extension); + Py_DECREF(iterator->second); + self->composite_fields->erase(iterator); } } return ClearFieldByDescriptor(self, descriptor); @@ -1739,13 +1809,16 @@ PyObject* ClearFieldByDescriptor( } PyObject* ClearField(CMessage* self, PyObject* arg) { - if (!PyString_Check(arg)) { + if (!(PyString_Check(arg) || PyUnicode_Check(arg))) { PyErr_SetString(PyExc_TypeError, "field name must be a string"); return NULL; } #if PY_MAJOR_VERSION < 3 - const char* field_name = PyString_AS_STRING(arg); - Py_ssize_t size = PyString_GET_SIZE(arg); + char* field_name; + Py_ssize_t size; + if (PyString_AsStringAndSize(arg, &field_name, &size) < 0) { + return NULL; + } #else Py_ssize_t size; const char* field_name = PyUnicode_AsUTF8AndSize(arg, &size); @@ -1770,14 +1843,16 @@ PyObject* ClearField(CMessage* self, PyObject* arg) { arg = arg_in_oneof.get(); } - // Release the field if it exists in the dict of composite fields. if (self->composite_fields) { - PyObject* value = PyDict_GetItem(self->composite_fields, arg); - if (value != NULL) { - if (InternalReleaseFieldByDescriptor(self, field_descriptor, value) < 0) { + CMessage::CompositeFieldsMap::iterator iterator = + self->composite_fields->find(field_descriptor); + if (iterator != self->composite_fields->end()) { + if (InternalReleaseFieldByDescriptor(self, field_descriptor, + iterator->second) < 0) { return NULL; } - PyDict_DelItem(self->composite_fields, arg); + Py_DECREF(iterator->second); + self->composite_fields->erase(iterator); } } return ClearFieldByDescriptor(self, field_descriptor); @@ -1787,9 +1862,18 @@ PyObject* Clear(CMessage* self) { AssureWritable(self); if (ForEachCompositeField(self, ReleaseChild(self)) == -1) return NULL; - Py_CLEAR(self->extensions); if (self->composite_fields) { - PyDict_Clear(self->composite_fields); + for (CMessage::CompositeFieldsMap::iterator it = + self->composite_fields->begin(); + it != self->composite_fields->end(); it++) { + Py_DECREF(it->second); + } + self->composite_fields->clear(); + } + if (self->unknown_field_set) { + unknown_fields::Clear( + reinterpret_cast<PyUnknownFields*>(self->unknown_field_set)); + Py_CLEAR(self->unknown_field_set); } self->message->Clear(); Py_RETURN_NONE; @@ -1946,7 +2030,7 @@ static PyObject* ToStr(CMessage* self) { PyObject* MergeFrom(CMessage* self, PyObject* arg) { CMessage* other_message; - if (!PyObject_TypeCheck(arg, &CMessage_Type)) { + if (!PyObject_TypeCheck(arg, CMessage_Type)) { PyErr_Format(PyExc_TypeError, "Parameter to MergeFrom() must be instance of same class: " "expected %s got %s.", @@ -1967,18 +2051,19 @@ PyObject* MergeFrom(CMessage* self, PyObject* arg) { } AssureWritable(self); - // TODO(tibell): Message::MergeFrom might turn some child Messages - // into mutable messages, invalidating the message field in the - // corresponding CMessages. We should run a FixupMessageReferences - // pass here. - self->message->MergeFrom(*other_message->message); + // Child message might be lazily created before MergeFrom. Make sure they + // are mutable at this point if child messages are really created. + if (ForEachCompositeField(self, FixupMessageAfterMerge(self)) == -1) { + return NULL; + } + Py_RETURN_NONE; } static PyObject* CopyFrom(CMessage* self, PyObject* arg) { CMessage* other_message; - if (!PyObject_TypeCheck(arg, &CMessage_Type)) { + if (!PyObject_TypeCheck(arg, CMessage_Type)) { PyErr_Format(PyExc_TypeError, "Parameter to CopyFrom() must be instance of same class: " "expected %s got %s.", @@ -2050,6 +2135,7 @@ static PyObject* MergeFromString(CMessage* self, PyObject* arg) { } AssureWritable(self); + io::CodedInputStream input( reinterpret_cast<const uint8*>(data), data_length); if (allow_oversize_protos) { @@ -2058,6 +2144,12 @@ static PyObject* MergeFromString(CMessage* self, PyObject* arg) { PyMessageFactory* factory = GetFactoryForMessage(self); input.SetExtensionRegistry(factory->pool->pool, factory->message_factory); bool success = self->message->MergePartialFromCodedStream(&input); + // Child message might be lazily created before MergeFrom. Make sure they + // are mutable at this point if child messages are really created. + if (ForEachCompositeField(self, FixupMessageAfterMerge(self)) == -1) { + return NULL; + } + if (success) { if (!input.ConsumedEntireMessage()) { // TODO(jieluo): Raise error and return NULL instead. @@ -2088,7 +2180,7 @@ PyObject* RegisterExtension(PyObject* cls, PyObject* extension_handle) { if (descriptor == NULL) { return NULL; } - if (!PyObject_TypeCheck(cls, &CMessageClass_Type)) { + if (!PyObject_TypeCheck(cls, CMessageClass_Type)) { PyErr_Format(PyExc_TypeError, "Expected a message class, got %s", cls->ob_type->tp_name); return NULL; @@ -2192,23 +2284,15 @@ static PyObject* ListFields(CMessage* self) { PyTuple_SET_ITEM(t.get(), 1, extension); } else { // Normal field - const string& field_name = fields[i]->name(); - ScopedPyObjectPtr py_field_name(PyString_FromStringAndSize( - field_name.c_str(), field_name.length())); - if (py_field_name == NULL) { - PyErr_SetString(PyExc_ValueError, "bad string"); - return NULL; - } ScopedPyObjectPtr field_descriptor( PyFieldDescriptor_FromDescriptor(fields[i])); if (field_descriptor == NULL) { return NULL; } - PyObject* field_value = - GetAttr(reinterpret_cast<PyObject*>(self), py_field_name.get()); + PyObject* field_value = GetFieldValue(self, fields[i]); if (field_value == NULL) { - PyErr_SetObject(PyExc_ValueError, py_field_name.get()); + PyErr_SetString(PyExc_ValueError, fields[i]->name().c_str()); return NULL; } PyTuple_SET_ITEM(t.get(), 0, field_descriptor.release()); @@ -2261,7 +2345,7 @@ static PyObject* RichCompare(CMessage* self, PyObject* other, int opid) { } bool equals = true; // If other is not a message, it cannot be equal. - if (!PyObject_TypeCheck(other, &CMessage_Type)) { + if (!PyObject_TypeCheck(other, CMessage_Type)) { equals = false; } const google::protobuf::Message* other_message = @@ -2277,6 +2361,7 @@ static PyObject* RichCompare(CMessage* self, PyObject* other, int opid) { *reinterpret_cast<CMessage*>(other)->message)) { equals = false; } + if (equals ^ (opid == Py_EQ)) { Py_RETURN_FALSE; } else { @@ -2498,7 +2583,7 @@ PyObject* DeepCopy(CMessage* self, PyObject* arg) { if (clone == NULL) { return NULL; } - if (!PyObject_TypeCheck(clone, &CMessage_Type)) { + if (!PyObject_TypeCheck(clone, CMessage_Type)) { Py_DECREF(clone); return NULL; } @@ -2592,26 +2677,29 @@ PyObject* _CheckCalledFromGeneratedFile(PyObject* unused, } static PyObject* GetExtensionDict(CMessage* self, void *closure) { - if (self->extensions) { - Py_INCREF(self->extensions); - return reinterpret_cast<PyObject*>(self->extensions); - } - // If there are extension_ranges, the message is "extendable". Allocate a // dictionary to store the extension fields. const Descriptor* descriptor = GetMessageDescriptor(Py_TYPE(self)); - if (descriptor->extension_range_count() > 0) { - ExtensionDict* extension_dict = extension_dict::NewExtensionDict(self); - if (extension_dict == NULL) { - return NULL; - } - self->extensions = extension_dict; - Py_INCREF(self->extensions); - return reinterpret_cast<PyObject*>(self->extensions); + if (!descriptor->extension_range_count()) { + PyErr_SetNone(PyExc_AttributeError); + return NULL; + } + if (!self->composite_fields) { + self->composite_fields = new CMessage::CompositeFieldsMap(); } + if (!self->composite_fields) { + return NULL; + } + ExtensionDict* extension_dict = extension_dict::NewExtensionDict(self); + return reinterpret_cast<PyObject*>(extension_dict); +} - PyErr_SetNone(PyExc_AttributeError); - return NULL; +static PyObject* UnknownFieldSet(CMessage* self) { + if (self->unknown_field_set == NULL) { + self->unknown_field_set = unknown_fields::NewPyUnknownFields(self); + } + Py_INCREF(self->unknown_field_set); + return self->unknown_field_set; } static PyObject* GetExtensionsByName(CMessage *self, void *closure) { @@ -2682,6 +2770,8 @@ static PyMethodDef Methods[] = { "Serializes the message to a string, only for initialized messages." }, { "SetInParent", (PyCFunction)SetInParent, METH_NOARGS, "Sets the has bit of the given field in its parent message." }, + { "UnknownFields", (PyCFunction)UnknownFieldSet, METH_NOARGS, + "Parse unknown field set"}, { "WhichOneof", (PyCFunction)WhichOneof, METH_O, "Returns the name of the field set inside a oneof, " "or None if no field is set." }, @@ -2693,30 +2783,53 @@ static PyMethodDef Methods[] = { { NULL, NULL} }; -static bool SetCompositeField( - CMessage* self, PyObject* name, PyObject* value) { +static bool SetCompositeField(CMessage* self, const FieldDescriptor* field, + PyObject* value) { if (self->composite_fields == NULL) { - self->composite_fields = PyDict_New(); - if (self->composite_fields == NULL) { - return false; - } + self->composite_fields = new CMessage::CompositeFieldsMap(); } - return PyDict_SetItem(self->composite_fields, name, value) == 0; + Py_INCREF(value); + Py_XDECREF((*self->composite_fields)[field]); + (*self->composite_fields)[field] = value; + return true; } PyObject* GetAttr(PyObject* pself, PyObject* name) { CMessage* self = reinterpret_cast<CMessage*>(pself); - PyObject* value = self->composite_fields ? - PyDict_GetItem(self->composite_fields, name) : NULL; - if (value != NULL) { - Py_INCREF(value); - return value; + PyObject* result = PyObject_GenericGetAttr( + reinterpret_cast<PyObject*>(self), name); + if (result != NULL) { + return result; + } + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + return NULL; } - const FieldDescriptor* field_descriptor = GetFieldDescriptor(self, name); - if (field_descriptor == NULL) { - return CMessage_Type.tp_base->tp_getattro( - reinterpret_cast<PyObject*>(self), name); + PyErr_Clear(); + return message_meta::GetClassAttribute( + CheckMessageClass(Py_TYPE(self)), name); +} + +PyObject* GetFieldValue(CMessage* self, + const FieldDescriptor* field_descriptor) { + if (self->composite_fields) { + CMessage::CompositeFieldsMap::iterator it = + self->composite_fields->find(field_descriptor); + if (it != self->composite_fields->end()) { + PyObject* value = it->second; + Py_INCREF(value); + return value; + } + } + + const Descriptor* message_descriptor = + (reinterpret_cast<CMessageClass*>(Py_TYPE(self)))->message_descriptor; + if (self->message->GetDescriptor() != field_descriptor->containing_type()) { + PyErr_Format(PyExc_TypeError, + "descriptor to field '%s' doesn't apply to '%s' object", + field_descriptor->full_name().c_str(), + Py_TYPE(self)->tp_name); + return NULL; } if (field_descriptor->is_map()) { @@ -2737,7 +2850,7 @@ PyObject* GetAttr(PyObject* pself, PyObject* name) { if (py_container == NULL) { return NULL; } - if (!SetCompositeField(self, name, py_container)) { + if (!SetCompositeField(self, field_descriptor, py_container)) { Py_DECREF(py_container); return NULL; } @@ -2761,7 +2874,7 @@ PyObject* GetAttr(PyObject* pself, PyObject* name) { if (py_container == NULL) { return NULL; } - if (!SetCompositeField(self, name, py_container)) { + if (!SetCompositeField(self, field_descriptor, py_container)) { Py_DECREF(py_container); return NULL; } @@ -2773,7 +2886,7 @@ PyObject* GetAttr(PyObject* pself, PyObject* name) { if (sub_message == NULL) { return NULL; } - if (!SetCompositeField(self, name, sub_message)) { + if (!SetCompositeField(self, field_descriptor, sub_message)) { Py_DECREF(sub_message); return NULL; } @@ -2783,44 +2896,35 @@ PyObject* GetAttr(PyObject* pself, PyObject* name) { return InternalGetScalar(self->message, field_descriptor); } -int SetAttr(PyObject* pself, PyObject* name, PyObject* value) { - CMessage* self = reinterpret_cast<CMessage*>(pself); - if (self->composite_fields && PyDict_Contains(self->composite_fields, name)) { - PyErr_SetString(PyExc_TypeError, "Can't set composite field"); +int SetFieldValue(CMessage* self, const FieldDescriptor* field_descriptor, + PyObject* value) { + if (self->message->GetDescriptor() != field_descriptor->containing_type()) { + PyErr_Format(PyExc_TypeError, + "descriptor to field '%s' doesn't apply to '%s' object", + field_descriptor->full_name().c_str(), + Py_TYPE(self)->tp_name); return -1; - } - - const FieldDescriptor* field_descriptor = GetFieldDescriptor(self, name); - if (field_descriptor != NULL) { + } else if (field_descriptor->label() == FieldDescriptor::LABEL_REPEATED) { + PyErr_Format(PyExc_AttributeError, + "Assignment not allowed to repeated " + "field \"%s\" in protocol message object.", + field_descriptor->name().c_str()); + return -1; + } else if (field_descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + PyErr_Format(PyExc_AttributeError, + "Assignment not allowed to " + "field \"%s\" in protocol message object.", + field_descriptor->name().c_str()); + return -1; + } else { AssureWritable(self); - if (field_descriptor->label() == FieldDescriptor::LABEL_REPEATED) { - PyErr_Format(PyExc_AttributeError, "Assignment not allowed to repeated " - "field \"%s\" in protocol message object.", - field_descriptor->name().c_str()); - return -1; - } else { - if (field_descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { - PyErr_Format(PyExc_AttributeError, "Assignment not allowed to " - "field \"%s\" in protocol message object.", - field_descriptor->name().c_str()); - return -1; - } else { - return InternalSetScalar(self, field_descriptor, value); - } - } + return InternalSetScalar(self, field_descriptor, value); } - - PyErr_Format(PyExc_AttributeError, - "Assignment not allowed " - "(no field \"%s\" in protocol message object).", - PyString_AsString(name)); - return -1; } - } // namespace cmessage -PyTypeObject CMessage_Type = { - PyVarObject_HEAD_INIT(&CMessageClass_Type, 0) +static CMessageClass _CMessage_Type = { { { + PyVarObject_HEAD_INIT(&_CMessageClass_Type, 0) FULL_MODULE_NAME ".CMessage", // tp_name sizeof(CMessage), // tp_basicsize 0, // tp_itemsize @@ -2837,9 +2941,10 @@ PyTypeObject CMessage_Type = { 0, // tp_call (reprfunc)cmessage::ToStr, // tp_str cmessage::GetAttr, // tp_getattro - cmessage::SetAttr, // tp_setattro + 0, // tp_setattro 0, // tp_as_buffer - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, // tp_flags + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE + | Py_TPFLAGS_HAVE_VERSION_TAG, // tp_flags "A ProtocolMessage", // tp_doc 0, // tp_traverse 0, // tp_clear @@ -2858,7 +2963,8 @@ PyTypeObject CMessage_Type = { (initproc)cmessage::Init, // tp_init 0, // tp_alloc cmessage::New, // tp_new -}; +} } }; +PyTypeObject* CMessage_Type = &_CMessage_Type.super.ht_type; // --- Exposing the C proto living inside Python proto to C code: @@ -2884,7 +2990,7 @@ static Message* MutableCProtoInsidePyProtoImpl(PyObject* msg) { } const Message* PyMessage_GetMessagePointer(PyObject* msg) { - if (!PyObject_TypeCheck(msg, &CMessage_Type)) { + if (!PyObject_TypeCheck(msg, CMessage_Type)) { PyErr_SetString(PyExc_TypeError, "Not a Message instance"); return NULL; } @@ -2893,15 +2999,14 @@ const Message* PyMessage_GetMessagePointer(PyObject* msg) { } Message* PyMessage_GetMutableMessagePointer(PyObject* msg) { - if (!PyObject_TypeCheck(msg, &CMessage_Type)) { + if (!PyObject_TypeCheck(msg, CMessage_Type)) { PyErr_SetString(PyExc_TypeError, "Not a Message instance"); return NULL; } + CMessage* cmsg = reinterpret_cast<CMessage*>(msg); - if ((cmsg->composite_fields && PyDict_Size(cmsg->composite_fields) != 0) || - (cmsg->extensions != NULL && - PyDict_Size(cmsg->extensions->values) != 0)) { + if (cmsg->composite_fields && !cmsg->composite_fields->empty()) { // There is currently no way of accurately syncing arbitrary changes to // the underlying C++ message back to the CMessage (e.g. removed repeated // composite containers). We only allow direct mutation of the underlying @@ -2945,22 +3050,29 @@ bool InitProto2MessageModule(PyObject *m) { // Initialize constants defined in this file. InitGlobals(); - CMessageClass_Type.tp_base = &PyType_Type; - if (PyType_Ready(&CMessageClass_Type) < 0) { + CMessageClass_Type->tp_base = &PyType_Type; + if (PyType_Ready(CMessageClass_Type) < 0) { return false; } PyModule_AddObject(m, "MessageMeta", - reinterpret_cast<PyObject*>(&CMessageClass_Type)); + reinterpret_cast<PyObject*>(CMessageClass_Type)); - if (PyType_Ready(&CMessage_Type) < 0) { + if (PyType_Ready(CMessage_Type) < 0) { + return false; + } + if (PyType_Ready(CFieldProperty_Type) < 0) { return false; } // DESCRIPTOR is set on each protocol buffer message class elsewhere, but set // it here as well to document that subclasses need to set it. - PyDict_SetItem(CMessage_Type.tp_dict, kDESCRIPTOR, Py_None); + PyDict_SetItem(CMessage_Type->tp_dict, kDESCRIPTOR, Py_None); + // Invalidate any cached data for the CMessage type. + // This call is necessary to correctly support Py_TPFLAGS_HAVE_VERSION_TAG, + // after we have modified CMessage_Type.tp_dict. + PyType_Modified(CMessage_Type); - PyModule_AddObject(m, "Message", reinterpret_cast<PyObject*>(&CMessage_Type)); + PyModule_AddObject(m, "Message", reinterpret_cast<PyObject*>(CMessage_Type)); // Initialize Repeated container types. { @@ -3003,6 +3115,22 @@ bool InitProto2MessageModule(PyObject *m) { } } + if (PyType_Ready(&PyUnknownFields_Type) < 0) { + return false; + } + + PyModule_AddObject(m, "UnknownFieldSet", + reinterpret_cast<PyObject*>( + &PyUnknownFields_Type)); + + if (PyType_Ready(&PyUnknownFieldRef_Type) < 0) { + return false; + } + + PyModule_AddObject(m, "UnknownField", + reinterpret_cast<PyObject*>( + &PyUnknownFieldRef_Type)); + // Initialize Map container types. if (!InitMapContainers()) { return false; diff --git a/python/google/protobuf/pyext/message.h b/python/google/protobuf/pyext/message.h index d754e62a..c112a88f 100644 --- a/python/google/protobuf/pyext/message.h +++ b/python/google/protobuf/pyext/message.h @@ -38,6 +38,7 @@ #include <memory> #include <string> +#include <unordered_map> #include <google/protobuf/stubs/common.h> #include <google/protobuf/pyext/thread_unsafe_shared_ptr.h> @@ -96,26 +97,25 @@ typedef struct CMessage { // made writable, at which point this field is set to false. bool read_only; - // A reference to a Python dictionary containing CMessage, + // A mapping indexed by field, containing CMessage, // RepeatedCompositeContainer, and RepeatedScalarContainer // objects. Used as a cache to make sure we don't have to make a // Python wrapper for the C++ Message objects on every access, or // deal with the synchronization nightmare that could create. - PyObject* composite_fields; + // Also cache extension fields. + // The FieldDescriptor is owned by the message's pool; PyObject references + // are owned. + typedef std::unordered_map<const FieldDescriptor*, PyObject*> + CompositeFieldsMap; + CompositeFieldsMap* composite_fields; - // A reference to the dictionary containing the message's extensions. - // Similar to composite_fields, acting as a cache, but also contains the - // required extension dict logic. - ExtensionDict* extensions; + // A reference to PyUnknownFields. + PyObject* unknown_field_set; // Implements the "weakref" protocol for this object. PyObject* weakreflist; } CMessage; -extern PyTypeObject CMessageClass_Type; -extern PyTypeObject CMessage_Type; - - // The (meta) type of all Messages classes. // It allows us to cache some C++ pointers in the class object itself, they are // faster to extract than from the type's dictionary. @@ -142,6 +142,8 @@ struct CMessageClass { } }; +extern PyTypeObject* CMessageClass_Type; +extern PyTypeObject* CMessage_Type; namespace cmessage { @@ -164,13 +166,13 @@ PyObject* InternalGetSubMessage( // Deletes a range of C++ submessages in a repeated field (following a // removal in a RepeatedCompositeContainer). // -// Releases messages to the provided cmessage_list if it is not NULL rather +// Releases submessages to the provided cmessage_list if it is not NULL rather // than just removing them from the underlying proto. This cmessage_list must // have a CMessage for each underlying submessage. The CMessages referred to // by slice will be removed from cmessage_list by this function. // // Corresponds to reflection api method RemoveLast. -int InternalDeleteRepeatedField(CMessage* self, +int InternalDeleteRepeatedField(Message* message, const FieldDescriptor* field_descriptor, PyObject* slice, PyObject* cmessage_list); @@ -235,15 +237,13 @@ PyObject* MergeFrom(CMessage* self, PyObject* arg); // has been registered with the same field number on this class. PyObject* RegisterExtension(PyObject* cls, PyObject* extension_handle); -// Retrieves an attribute named 'name' from 'self', which is interpreted as a -// CMessage. Returns the attribute value on success, or null on failure. -// -// Returns a new reference. -PyObject* GetAttr(PyObject* self, PyObject* name); - -// Set the value of the attribute named 'name', for 'self', which is interpreted -// as a CMessage, to the value 'value'. Returns -1 on failure. -int SetAttr(PyObject* self, PyObject* name, PyObject* value); +// Get a field from a message. +PyObject* GetFieldValue(CMessage* self, + const FieldDescriptor* field_descriptor); +// Sets the value of a scalar field in a message. +// On error, return -1 with an extension set. +int SetFieldValue(CMessage* self, const FieldDescriptor* field_descriptor, + PyObject* value); PyObject* FindInitializationErrors(CMessage* self); @@ -332,7 +332,7 @@ bool CheckAndSetString( bool append, int index); PyObject* ToStringObject(const FieldDescriptor* descriptor, - const string& value); + const std::string& value); // Check if the passed field descriptor belongs to the given message. // If not, return false and set a Python exception (a KeyError) @@ -357,6 +357,6 @@ extern template bool CheckAndGetInteger<uint64>(PyObject*, uint64*); } // namespace python } // namespace protobuf - } // namespace google + #endif // GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_H__ diff --git a/python/google/protobuf/pyext/message_factory.cc b/python/google/protobuf/pyext/message_factory.cc index bacc76a6..efaa2617 100644 --- a/python/google/protobuf/pyext/message_factory.cc +++ b/python/google/protobuf/pyext/message_factory.cc @@ -28,6 +28,8 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include <unordered_map> + #include <Python.h> #include <google/protobuf/dynamic_message.h> @@ -137,7 +139,7 @@ CMessageClass* GetOrCreateMessageClass(PyMessageFactory* self, // This is the same implementation as MessageFactory.GetPrototype(). // Do not create a MessageClass that already exists. - hash_map<const Descriptor*, CMessageClass*>::iterator it = + std::unordered_map<const Descriptor*, CMessageClass*>::iterator it = self->classes_by_descriptor->find(descriptor); if (it != self->classes_by_descriptor->end()) { Py_INCREF(it->second); @@ -158,7 +160,7 @@ CMessageClass* GetOrCreateMessageClass(PyMessageFactory* self, return NULL; } ScopedPyObjectPtr message_class(PyObject_CallObject( - reinterpret_cast<PyObject*>(&CMessageClass_Type), args.get())); + reinterpret_cast<PyObject*>(CMessageClass_Type), args.get())); if (message_class == NULL) { return NULL; } diff --git a/python/google/protobuf/pyext/message_factory.h b/python/google/protobuf/pyext/message_factory.h index 36092f7e..06444b0a 100644 --- a/python/google/protobuf/pyext/message_factory.h +++ b/python/google/protobuf/pyext/message_factory.h @@ -33,7 +33,7 @@ #include <Python.h> -#include <google/protobuf/stubs/hash.h> +#include <unordered_map> #include <google/protobuf/descriptor.h> #include <google/protobuf/pyext/descriptor_pool.h> @@ -66,7 +66,8 @@ struct PyMessageFactory { // // Descriptor pointers stored here are owned by the DescriptorPool above. // Python references to classes are owned by this PyDescriptorPool. - typedef hash_map<const Descriptor*, CMessageClass*> ClassesByMessageMap; + typedef std::unordered_map<const Descriptor*, CMessageClass*> + ClassesByMessageMap; ClassesByMessageMap* classes_by_descriptor; }; @@ -98,6 +99,6 @@ bool InitMessageFactory(); } // namespace python } // namespace protobuf - } // namespace google + #endif // GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_FACTORY_H__ diff --git a/python/google/protobuf/pyext/message_module.cc b/python/google/protobuf/pyext/message_module.cc index 8c933866..8d465eb5 100644 --- a/python/google/protobuf/pyext/message_module.cc +++ b/python/google/protobuf/pyext/message_module.cc @@ -39,39 +39,16 @@ namespace { // C++ API. Clients get at this via proto_api.h struct ApiImplementation : google::protobuf::python::PyProto_API { - const google::protobuf::Message* - GetMessagePointer(PyObject* msg) const override { + const google::protobuf::Message* GetMessagePointer(PyObject* msg) const override { return google::protobuf::python::PyMessage_GetMessagePointer(msg); } - google::protobuf::Message* - GetMutableMessagePointer(PyObject* msg) const override { + google::protobuf::Message* GetMutableMessagePointer(PyObject* msg) const override { return google::protobuf::python::PyMessage_GetMutableMessagePointer(msg); } }; } // namespace -static PyObject* GetPythonProto3PreserveUnknownsDefault( - PyObject* /*m*/, PyObject* /*args*/) { - if (google::protobuf::internal::GetProto3PreserveUnknownsDefault()) { - Py_RETURN_TRUE; - } else { - Py_RETURN_FALSE; - } -} - -static PyObject* SetPythonProto3PreserveUnknownsDefault( - PyObject* /*m*/, PyObject* arg) { - if (!arg || !PyBool_Check(arg)) { - PyErr_SetString( - PyExc_TypeError, - "Argument to SetPythonProto3PreserveUnknownsDefault must be boolean"); - return NULL; - } - google::protobuf::internal::SetProto3PreserveUnknownsDefault(PyObject_IsTrue(arg)); - Py_RETURN_NONE; -} - static const char module_docstring[] = "python-proto2 is a module that can be used to enhance proto2 Python API\n" "performance.\n" @@ -84,13 +61,6 @@ static PyMethodDef ModuleMethods[] = { (PyCFunction)google::protobuf::python::cmessage::SetAllowOversizeProtos, METH_O, "Enable/disable oversize proto parsing."}, // DO NOT USE: For migration and testing only. - {"GetPythonProto3PreserveUnknownsDefault", - (PyCFunction)GetPythonProto3PreserveUnknownsDefault, - METH_NOARGS, "Get Proto3 preserve unknowns default."}, - // DO NOT USE: For migration and testing only. - {"SetPythonProto3PreserveUnknownsDefault", - (PyCFunction)SetPythonProto3PreserveUnknownsDefault, - METH_O, "Enable/disable proto3 unknowns preservation."}, { NULL, NULL} }; @@ -113,35 +83,32 @@ static struct PyModuleDef _module = { #define INITFUNC_ERRORVAL #endif -extern "C" { - PyMODINIT_FUNC INITFUNC(void) { - PyObject* m; +PyMODINIT_FUNC INITFUNC() { + PyObject* m; #if PY_MAJOR_VERSION >= 3 - m = PyModule_Create(&_module); + m = PyModule_Create(&_module); #else - m = Py_InitModule3("_message", ModuleMethods, - module_docstring); + m = Py_InitModule3("_message", ModuleMethods, module_docstring); #endif - if (m == NULL) { - return INITFUNC_ERRORVAL; - } + if (m == NULL) { + return INITFUNC_ERRORVAL; + } - if (!google::protobuf::python::InitProto2MessageModule(m)) { - Py_DECREF(m); - return INITFUNC_ERRORVAL; - } - - // Adds the C++ API - if (PyObject* api = - PyCapsule_New(new ApiImplementation(), - google::protobuf::python::PyProtoAPICapsuleName(), NULL)) { - PyModule_AddObject(m, "proto_API", api); - } else { - return INITFUNC_ERRORVAL; - } + if (!google::protobuf::python::InitProto2MessageModule(m)) { + Py_DECREF(m); + return INITFUNC_ERRORVAL; + } + + // Adds the C++ API + if (PyObject* api = + PyCapsule_New(new ApiImplementation(), + google::protobuf::python::PyProtoAPICapsuleName(), NULL)) { + PyModule_AddObject(m, "proto_API", api); + } else { + return INITFUNC_ERRORVAL; + } #if PY_MAJOR_VERSION >= 3 - return m; + return m; #endif - } } diff --git a/python/google/protobuf/pyext/repeated_composite_container.cc b/python/google/protobuf/pyext/repeated_composite_container.cc index 5874d5de..ca700580 100644 --- a/python/google/protobuf/pyext/repeated_composite_container.cc +++ b/python/google/protobuf/pyext/repeated_composite_container.cc @@ -61,9 +61,9 @@ namespace repeated_composite_container { // TODO(tibell): We might also want to check: // GOOGLE_CHECK_NOTNULL((self)->owner.get()); -#define GOOGLE_CHECK_ATTACHED(self) \ - do { \ - GOOGLE_CHECK_NOTNULL((self)->message); \ +#define GOOGLE_CHECK_ATTACHED(self) \ + do { \ + GOOGLE_CHECK_NOTNULL((self)->message); \ GOOGLE_CHECK_NOTNULL((self)->parent_field_descriptor); \ } while (0); @@ -152,6 +152,8 @@ static PyObject* AddToAttached(RepeatedCompositeContainer* self, cmsg->message = sub_message; cmsg->parent = self->parent; if (cmessage::InitAttributes(cmsg, args, kwargs) < 0) { + message->GetReflection()->RemoveLast( + message, self->parent_field_descriptor); Py_DECREF(cmsg); return NULL; } @@ -210,7 +212,7 @@ PyObject* Extend(RepeatedCompositeContainer* self, PyObject* value) { } ScopedPyObjectPtr next; while ((next.reset(PyIter_Next(iter.get()))) != NULL) { - if (!PyObject_TypeCheck(next.get(), &CMessage_Type)) { + if (!PyObject_TypeCheck(next.get(), CMessage_Type)) { PyErr_SetString(PyExc_TypeError, "Not a cmessage"); return NULL; } @@ -270,8 +272,8 @@ int AssignSubscript(RepeatedCompositeContainer* self, } // Delete from the underlying Message, if any. - if (self->parent != NULL) { - if (cmessage::InternalDeleteRepeatedField(self->parent, + if (self->message != nullptr) { + if (cmessage::InternalDeleteRepeatedField(self->message, self->parent_field_descriptor, slice, self->child_messages) < 0) { @@ -484,15 +486,15 @@ static PyObject* Pop(PyObject* pself, PyObject* args) { } // Release field of parent message and transfer the ownership to target. -void ReleaseLastTo(CMessage* parent, +void ReleaseLastTo(Message* message, const FieldDescriptor* field, CMessage* target) { - GOOGLE_CHECK_NOTNULL(parent); - GOOGLE_CHECK_NOTNULL(field); - GOOGLE_CHECK_NOTNULL(target); + GOOGLE_CHECK(message != nullptr); + GOOGLE_CHECK(field != nullptr); + GOOGLE_CHECK(target != nullptr); CMessage::OwnerRef released_message( - parent->message->GetReflection()->ReleaseLast(parent->message, field)); + message->GetReflection()->ReleaseLast(message, field)); // TODO(tibell): Deal with proto1. target->parent = NULL; @@ -522,7 +524,7 @@ int Release(RepeatedCompositeContainer* self) { for (Py_ssize_t i = size - 1; i >= 0; --i) { CMessage* child_cmessage = reinterpret_cast<CMessage*>( PyList_GET_ITEM(self->child_messages, i)); - ReleaseLastTo(self->parent, field, child_cmessage); + ReleaseLastTo(message, field, child_cmessage); } // Detach from containing message. diff --git a/python/google/protobuf/pyext/repeated_composite_container.h b/python/google/protobuf/pyext/repeated_composite_container.h index e5e946aa..d0755771 100644 --- a/python/google/protobuf/pyext/repeated_composite_container.h +++ b/python/google/protobuf/pyext/repeated_composite_container.h @@ -154,13 +154,13 @@ int SetOwner(RepeatedCompositeContainer* self, // Message to 'target'. // // Corresponds to reflection api method ReleaseMessage. -void ReleaseLastTo(CMessage* parent, +void ReleaseLastTo(Message* message, const FieldDescriptor* field, CMessage* target); } // namespace repeated_composite_container } // namespace python } // namespace protobuf - } // namespace google + #endif // GOOGLE_PROTOBUF_PYTHON_CPP_REPEATED_COMPOSITE_CONTAINER_H__ diff --git a/python/google/protobuf/pyext/repeated_scalar_container.cc b/python/google/protobuf/pyext/repeated_scalar_container.cc index de3b6e14..ac06cff3 100644 --- a/python/google/protobuf/pyext/repeated_scalar_container.cc +++ b/python/google/protobuf/pyext/repeated_scalar_container.cc @@ -104,7 +104,8 @@ static int AssignItem(PyObject* pself, Py_ssize_t index, PyObject* arg) { if (arg == NULL) { ScopedPyObjectPtr py_index(PyLong_FromLong(index)); - return cmessage::InternalDeleteRepeatedField(self->parent, field_descriptor, + return cmessage::InternalDeleteRepeatedField(self->message, + field_descriptor, py_index.get(), NULL); } @@ -467,7 +468,7 @@ static int AssSubscript(PyObject* pself, PyObject* slice, PyObject* value) { if (value == NULL) { return cmessage::InternalDeleteRepeatedField( - self->parent, field_descriptor, slice, NULL); + self->message, field_descriptor, slice, nullptr); } if (!create_list) { @@ -663,6 +664,10 @@ static PyObject* ToStr(PyObject* pself) { return PyObject_Repr(list.get()); } +static PyObject* MergeFrom(PyObject* pself, PyObject* arg) { + return Extend(reinterpret_cast<RepeatedScalarContainer*>(pself), arg); +} + // The private constructor of RepeatedScalarContainer objects. PyObject *NewContainer( CMessage* parent, const FieldDescriptor* parent_field_descriptor) { @@ -776,6 +781,8 @@ static PyMethodDef Methods[] = { "Removes an object from the repeated container." }, { "sort", (PyCFunction)Sort, METH_VARARGS | METH_KEYWORDS, "Sorts the repeated container."}, + { "MergeFrom", (PyCFunction)MergeFrom, METH_O, + "Merges a repeated container into the current container." }, { NULL, NULL } }; diff --git a/python/google/protobuf/pyext/repeated_scalar_container.h b/python/google/protobuf/pyext/repeated_scalar_container.h index 559dec98..4dcecbac 100644 --- a/python/google/protobuf/pyext/repeated_scalar_container.h +++ b/python/google/protobuf/pyext/repeated_scalar_container.h @@ -104,6 +104,6 @@ void SetOwner(RepeatedScalarContainer* self, } // namespace repeated_scalar_container } // namespace python } // namespace protobuf - } // namespace google + #endif // GOOGLE_PROTOBUF_PYTHON_CPP_REPEATED_SCALAR_CONTAINER_H__ diff --git a/python/google/protobuf/pyext/safe_numerics.h b/python/google/protobuf/pyext/safe_numerics.h index 639ba2c8..93ae640e 100644 --- a/python/google/protobuf/pyext/safe_numerics.h +++ b/python/google/protobuf/pyext/safe_numerics.h @@ -132,10 +132,10 @@ template <class Dest, class Source> inline bool IsValidNumericCast(Source source) { typedef std::numeric_limits<Source> SourceLimits; typedef std::numeric_limits<Dest> DestLimits; - GOOGLE_COMPILE_ASSERT(SourceLimits::is_specialized, argument_must_be_numeric); - GOOGLE_COMPILE_ASSERT(SourceLimits::is_integer, argument_must_be_integral); - GOOGLE_COMPILE_ASSERT(DestLimits::is_specialized, result_must_be_numeric); - GOOGLE_COMPILE_ASSERT(DestLimits::is_integer, result_must_be_integral); + static_assert(SourceLimits::is_specialized, "argument must be numeric"); + static_assert(SourceLimits::is_integer, "argument must be integral"); + static_assert(DestLimits::is_specialized, "result must be numeric"); + static_assert(DestLimits::is_integer, "result must be integral"); return IsValidNumericCastImpl< sizeof(Dest) == sizeof(Source), @@ -150,7 +150,7 @@ inline bool IsValidNumericCast(Source source) { // checked_numeric_cast<> is analogous to static_cast<> for numeric types, // except that it CHECKs that the specified numeric conversion will not // overflow or underflow. Floating point arguments are not currently allowed -// (this is COMPILE_ASSERTd), though this could be supported if necessary. +// (this is static_asserted), though this could be supported if necessary. template <class Dest, class Source> inline Dest checked_numeric_cast(Source source) { GOOGLE_CHECK(IsValidNumericCast<Dest>(source)); @@ -159,6 +159,6 @@ inline Dest checked_numeric_cast(Source source) { } // namespace python } // namespace protobuf - } // namespace google + #endif // GOOGLE_PROTOBUF_PYTHON_CPP_SAFE_NUMERICS_H__ diff --git a/python/google/protobuf/pyext/thread_unsafe_shared_ptr.h b/python/google/protobuf/pyext/thread_unsafe_shared_ptr.h index ad804b5f..79fa9e3d 100644 --- a/python/google/protobuf/pyext/thread_unsafe_shared_ptr.h +++ b/python/google/protobuf/pyext/thread_unsafe_shared_ptr.h @@ -99,6 +99,6 @@ class ThreadUnsafeSharedPtr { } // namespace python } // namespace protobuf - } // namespace google + #endif // GOOGLE_PROTOBUF_PYTHON_CPP_THREAD_UNSAFE_SHARED_PTR_H__ diff --git a/python/google/protobuf/pyext/unknown_fields.cc b/python/google/protobuf/pyext/unknown_fields.cc new file mode 100755 index 00000000..760452f2 --- /dev/null +++ b/python/google/protobuf/pyext/unknown_fields.cc @@ -0,0 +1,355 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <google/protobuf/pyext/unknown_fields.h> + +#include <Python.h> +#include <set> +#include <memory> + +#include <google/protobuf/message.h> +#include <google/protobuf/pyext/message.h> +#include <google/protobuf/pyext/scoped_pyobject_ptr.h> +#include <google/protobuf/unknown_field_set.h> +#include <google/protobuf/wire_format_lite.h> + +#if PY_MAJOR_VERSION >= 3 + #define PyInt_FromLong PyLong_FromLong +#endif + +namespace google { +namespace protobuf { +namespace python { + +namespace unknown_fields { + +static Py_ssize_t Len(PyObject* pself) { + PyUnknownFields* self = + reinterpret_cast<PyUnknownFields*>(pself); + if (self->fields == NULL) { + PyErr_Format(PyExc_ValueError, + "UnknownFields does not exist. " + "The parent message might be cleared."); + return -1; + } + return self->fields->field_count(); +} + +void Clear(PyUnknownFields* self) { + for (std::set<PyUnknownFields*>::iterator it = + self->sub_unknown_fields.begin(); + it != self->sub_unknown_fields.end(); it++) { + Clear(*it); + } + self->fields = NULL; + self->sub_unknown_fields.clear(); +} + +PyObject* NewPyUnknownFieldRef(PyUnknownFields* parent, + Py_ssize_t index); + +static PyObject* Item(PyObject* pself, Py_ssize_t index) { + PyUnknownFields* self = + reinterpret_cast<PyUnknownFields*>(pself); + if (self->fields == NULL) { + PyErr_Format(PyExc_ValueError, + "UnknownFields does not exist. " + "The parent message might be cleared."); + return NULL; + } + Py_ssize_t total_size = self->fields->field_count(); + if (index < 0) { + index = total_size + index; + } + if (index < 0 || index >= total_size) { + PyErr_Format(PyExc_IndexError, + "index (%zd) out of range", + index); + return NULL; + } + + return unknown_fields::NewPyUnknownFieldRef(self, index); +} + +PyObject* NewPyUnknownFields(CMessage* c_message) { + PyUnknownFields* self = reinterpret_cast<PyUnknownFields*>( + PyType_GenericAlloc(&PyUnknownFields_Type, 0)); + if (self == NULL) { + return NULL; + } + // Call "placement new" to initialize PyUnknownFields. + new (self) PyUnknownFields; + + Py_INCREF(c_message); + self->parent = reinterpret_cast<PyObject*>(c_message); + Message* message = c_message->message; + const Reflection* reflection = message->GetReflection(); + self->fields = &reflection->GetUnknownFields(*message); + + return reinterpret_cast<PyObject*>(self); +} + +PyObject* NewPyUnknownFieldRef(PyUnknownFields* parent, + Py_ssize_t index) { + PyUnknownFieldRef* self = reinterpret_cast<PyUnknownFieldRef*>( + PyType_GenericAlloc(&PyUnknownFieldRef_Type, 0)); + if (self == NULL) { + return NULL; + } + + Py_INCREF(parent); + self->parent = parent; + self->index = index; + + return reinterpret_cast<PyObject*>(self); +} + +static void Dealloc(PyObject* pself) { + PyUnknownFields* self = + reinterpret_cast<PyUnknownFields*>(pself); + if (PyObject_TypeCheck(self->parent, &PyUnknownFields_Type)) { + reinterpret_cast<PyUnknownFields*>( + self->parent)->sub_unknown_fields.erase(self); + } + Py_CLEAR(self->parent); + self->~PyUnknownFields(); +} + +static PySequenceMethods SqMethods = { + Len, /* sq_length */ + 0, /* sq_concat */ + 0, /* sq_repeat */ + Item, /* sq_item */ + 0, /* sq_slice */ + 0, /* sq_ass_item */ +}; + +} // namespace unknown_fields + +PyTypeObject PyUnknownFields_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + FULL_MODULE_NAME ".PyUnknownFields", // tp_name + sizeof(PyUnknownFields), // tp_basicsize + 0, // tp_itemsize + unknown_fields::Dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + &unknown_fields::SqMethods, // tp_as_sequence + 0, // tp_as_mapping + PyObject_HashNotImplemented, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "unknown field set", // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + 0, // tp_methods + 0, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init +}; + +namespace unknown_field { +static PyObject* PyUnknownFields_FromUnknownFieldSet( + PyUnknownFields* parent, const UnknownFieldSet& fields) { + PyUnknownFields* self = reinterpret_cast<PyUnknownFields*>( + PyType_GenericAlloc(&PyUnknownFields_Type, 0)); + if (self == NULL) { + return NULL; + } + // Call "placement new" to initialize PyUnknownFields. + new (self) PyUnknownFields; + + Py_INCREF(parent); + self->parent = reinterpret_cast<PyObject*>(parent); + self->fields = &fields; + parent->sub_unknown_fields.emplace(self); + + return reinterpret_cast<PyObject*>(self); +} + +const UnknownField* GetUnknownField(PyUnknownFieldRef* self) { + const UnknownFieldSet* fields = self->parent->fields; + if (fields == NULL) { + PyErr_Format(PyExc_ValueError, + "UnknownField does not exist. " + "The parent message might be cleared."); + return NULL; + } + ssize_t total_size = fields->field_count(); + if (self->index >= total_size) { + PyErr_Format(PyExc_ValueError, + "UnknownField does not exist. " + "The parent message might be cleared."); + return NULL; + } + return &fields->field(self->index); +} + +static PyObject* GetFieldNumber(PyUnknownFieldRef* self, void *closure) { + const UnknownField* unknown_field = GetUnknownField(self); + if (unknown_field == NULL) { + return NULL; + } + return PyInt_FromLong(unknown_field->number()); +} + +using internal::WireFormatLite; +static PyObject* GetWireType(PyUnknownFieldRef* self, void *closure) { + const UnknownField* unknown_field = GetUnknownField(self); + if (unknown_field == NULL) { + return NULL; + } + + // Assign a default value to suppress may-unintialized warnings (errors + // when built in some places). + WireFormatLite::WireType wire_type = WireFormatLite::WIRETYPE_VARINT; + switch (unknown_field->type()) { + case UnknownField::TYPE_VARINT: + wire_type = WireFormatLite::WIRETYPE_VARINT; + break; + case UnknownField::TYPE_FIXED32: + wire_type = WireFormatLite::WIRETYPE_FIXED32; + break; + case UnknownField::TYPE_FIXED64: + wire_type = WireFormatLite::WIRETYPE_FIXED64; + break; + case UnknownField::TYPE_LENGTH_DELIMITED: + wire_type = WireFormatLite::WIRETYPE_LENGTH_DELIMITED; + break; + case UnknownField::TYPE_GROUP: + wire_type = WireFormatLite::WIRETYPE_START_GROUP; + break; + } + return PyInt_FromLong(wire_type); +} + +static PyObject* GetData(PyUnknownFieldRef* self, void *closure) { + const UnknownField* field = GetUnknownField(self); + if (field == NULL) { + return NULL; + } + PyObject* data = NULL; + switch (field->type()) { + case UnknownField::TYPE_VARINT: + data = PyInt_FromLong(field->varint()); + break; + case UnknownField::TYPE_FIXED32: + data = PyInt_FromLong(field->fixed32()); + break; + case UnknownField::TYPE_FIXED64: + data = PyInt_FromLong(field->fixed64()); + break; + case UnknownField::TYPE_LENGTH_DELIMITED: + data = PyBytes_FromStringAndSize(field->length_delimited().data(), + field->GetLengthDelimitedSize()); + break; + case UnknownField::TYPE_GROUP: + data = PyUnknownFields_FromUnknownFieldSet( + self->parent, field->group()); + break; + } + return data; +} + +static void Dealloc(PyObject* pself) { + PyUnknownFieldRef* self = + reinterpret_cast<PyUnknownFieldRef*>(pself); + Py_CLEAR(self->parent); +} + +static PyGetSetDef Getters[] = { + {"field_number", (getter)GetFieldNumber, NULL}, + {"wire_type", (getter)GetWireType, NULL}, + {"data", (getter)GetData, NULL}, + {NULL} +}; + +} // namespace unknown_field + +PyTypeObject PyUnknownFieldRef_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + FULL_MODULE_NAME ".PyUnknownFieldRef", // tp_name + sizeof(PyUnknownFieldRef), // tp_basicsize + 0, // tp_itemsize + unknown_field::Dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + PyObject_HashNotImplemented, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "unknown field", // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + 0, // tp_methods + 0, // tp_members + unknown_field::Getters, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init +}; + + +} // namespace python +} // namespace protobuf +} // namespace google diff --git a/python/google/protobuf/pyext/unknown_fields.h b/python/google/protobuf/pyext/unknown_fields.h new file mode 100755 index 00000000..94d55e14 --- /dev/null +++ b/python/google/protobuf/pyext/unknown_fields.h @@ -0,0 +1,90 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_UNKNOWN_FIELDS_H__ +#define GOOGLE_PROTOBUF_PYTHON_CPP_UNKNOWN_FIELDS_H__ + +#include <Python.h> + +#include <memory> +#include <set> + +#include <google/protobuf/pyext/message.h> + +namespace google { +namespace protobuf { + +class UnknownField; +class UnknownFieldSet; + +namespace python { +struct CMessage; + +typedef struct PyUnknownFields { + PyObject_HEAD; + // Strong pointer to the parent CMessage or PyUnknownFields. + // The top PyUnknownFields holds a reference to its parent CMessage + // object before release. + // Sub PyUnknownFields holds reference to parent PyUnknownFields. + PyObject* parent; + + // Pointer to the C++ UnknownFieldSet. + // PyUnknownFields does not own this pointer. + const UnknownFieldSet* fields; + + // Weak references to child unknown fields. + std::set<PyUnknownFields*> sub_unknown_fields; +} PyUnknownFields; + +typedef struct PyUnknownFieldRef { + PyObject_HEAD; + // Every Python PyUnknownFieldRef holds a reference to its parent + // PyUnknownFields in order to keep it alive. + PyUnknownFields* parent; + + // The UnknownField index in UnknownFields. + Py_ssize_t index; +} UknownFieldRef; + +extern PyTypeObject PyUnknownFields_Type; +extern PyTypeObject PyUnknownFieldRef_Type; + +namespace unknown_fields { + +// Builds an PyUnknownFields for a specific message. +PyObject* NewPyUnknownFields(CMessage *parent); +void Clear(PyUnknownFields* self); + +} // namespace unknown_fields +} // namespace python +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_PYTHON_CPP_UNKNOWN_FIELDS_H__ |