diff options
author | Feng Xiao <xfxyjwf@gmail.com> | 2015-08-23 17:50:38 -0700 |
---|---|---|
committer | Feng Xiao <xfxyjwf@gmail.com> | 2015-08-23 17:50:38 -0700 |
commit | b17ec3ca11ed13cc0d984f6d8be112c246b1994d (patch) | |
tree | c7c05b5d36fdb3c0a601a4dba0763e1e08f7bc52 /python | |
parent | eee38b0c018b3279f77d03dff796f440f40d3516 (diff) | |
download | protobuf-b17ec3ca11ed13cc0d984f6d8be112c246b1994d.tar.gz protobuf-b17ec3ca11ed13cc0d984f6d8be112c246b1994d.tar.bz2 protobuf-b17ec3ca11ed13cc0d984f6d8be112c246b1994d.zip |
Down-integrate from internal code base.
Diffstat (limited to 'python')
-rw-r--r-- | python/google/protobuf/pyext/descriptor.cc | 66 | ||||
-rw-r--r-- | python/google/protobuf/pyext/descriptor_pool.cc | 46 | ||||
-rw-r--r-- | python/google/protobuf/pyext/descriptor_pool.h | 15 |
3 files changed, 99 insertions, 28 deletions
diff --git a/python/google/protobuf/pyext/descriptor.cc b/python/google/protobuf/pyext/descriptor.cc index 8581f529..3806643f 100644 --- a/python/google/protobuf/pyext/descriptor.cc +++ b/python/google/protobuf/pyext/descriptor.cc @@ -62,6 +62,14 @@ namespace google { namespace protobuf { namespace python { +// Store interned descriptors, so that the same C++ descriptor yields the same +// Python object. Objects are not immortal: this map does not own the +// references, and items are deleted when the last reference to the object is +// released. +// This is enough to support the "is" operator on live objects. +// All descriptors are stored here. +hash_map<const void*, PyObject*> interned_descriptors; + PyObject* PyString_FromCppString(const string& str) { return PyString_FromStringAndSize(str.c_str(), str.size()); } @@ -147,6 +155,24 @@ static int CheckCalledFromGeneratedFile(const char* attr_name) { // Helper functions for descriptor objects. +// A set of templates to retrieve the C++ FileDescriptor of any descriptor. +template<class DescriptorClass> +const FileDescriptor* GetFileDescriptor(const DescriptorClass* descriptor) { + return descriptor->file(); +} +template<> +const FileDescriptor* GetFileDescriptor(const FileDescriptor* descriptor) { + return descriptor; +} +template<> +const FileDescriptor* GetFileDescriptor(const EnumValueDescriptor* descriptor) { + return descriptor->type()->file(); +} +template<> +const FileDescriptor* GetFileDescriptor(const OneofDescriptor* descriptor) { + return descriptor->containing_type()->file(); +} + // Converts options into a Python protobuf, and cache the result. // // This is a bit tricky because options can contain extension fields defined in @@ -156,8 +182,13 @@ static int CheckCalledFromGeneratedFile(const char* attr_name) { // Always returns a new reference. template<class DescriptorClass> static PyObject* GetOrBuildOptions(const DescriptorClass *descriptor) { + // Options (and their extensions) are completely resolved in the proto file + // containing the descriptor. + PyDescriptorPool* pool = GetDescriptorPool_FromPool( + GetFileDescriptor(descriptor)->pool()); + hash_map<const void*, PyObject*>* descriptor_options = - GetDescriptorPool()->descriptor_options; + pool->descriptor_options; // First search in the cache. if (descriptor_options->find(descriptor) != descriptor_options->end()) { PyObject *value = (*descriptor_options)[descriptor]; @@ -170,7 +201,7 @@ static PyObject* GetOrBuildOptions(const DescriptorClass *descriptor) { const Message& options(descriptor->options()); const Descriptor *message_type = options.GetDescriptor(); PyObject* message_class(cdescriptor_pool::GetMessageClass( - GetDescriptorPool(), message_type)); + pool, message_type)); if (message_class == NULL) { PyErr_Format(PyExc_TypeError, "Could not retrieve class for Options: %s", message_type->full_name().c_str()); @@ -192,7 +223,7 @@ static PyObject* GetOrBuildOptions(const DescriptorClass *descriptor) { options.SerializeToString(&serialized); io::CodedInputStream input( reinterpret_cast<const uint8*>(serialized.c_str()), serialized.size()); - input.SetExtensionRegistry(GetDescriptorPool()->pool, + input.SetExtensionRegistry(pool->pool, GetDescriptorPool()->message_factory); bool success = cmsg->message->MergePartialFromCodedStream(&input); if (!success) { @@ -203,7 +234,7 @@ static PyObject* GetOrBuildOptions(const DescriptorClass *descriptor) { // Cache the result. Py_INCREF(value); - (*GetDescriptorPool()->descriptor_options)[descriptor] = value.get(); + (*pool->descriptor_options)[descriptor] = value.get(); return value.release(); } @@ -237,6 +268,9 @@ typedef struct PyBaseDescriptor { // Pointer to the C++ proto2 descriptor. // Like all descriptors, it is owned by the global DescriptorPool. const void* descriptor; + + // Owned reference to the DescriptorPool, to ensure it is kept alive. + PyDescriptorPool* pool; } PyBaseDescriptor; @@ -258,7 +292,9 @@ namespace descriptor { // 'was_created' is an optional pointer to a bool, and is set to true if a new // object was allocated. // Always return a new reference. -PyObject* NewInternedDescriptor(PyTypeObject* type, const void* descriptor, +template<class DescriptorClass> +PyObject* NewInternedDescriptor(PyTypeObject* type, + const DescriptorClass* descriptor, bool* was_created) { if (was_created) { *was_created = false; @@ -270,8 +306,8 @@ PyObject* NewInternedDescriptor(PyTypeObject* type, const void* descriptor, // See if the object is in the map of interned descriptors hash_map<const void*, PyObject*>::iterator it = - GetDescriptorPool()->interned_descriptors->find(descriptor); - if (it != GetDescriptorPool()->interned_descriptors->end()) { + interned_descriptors.find(descriptor); + if (it != interned_descriptors.end()) { GOOGLE_DCHECK(Py_TYPE(it->second) == type); Py_INCREF(it->second); return it->second; @@ -283,10 +319,21 @@ PyObject* NewInternedDescriptor(PyTypeObject* type, const void* descriptor, return NULL; } py_descriptor->descriptor = descriptor; + // and cache it. - GetDescriptorPool()->interned_descriptors->insert( + interned_descriptors.insert( std::make_pair(descriptor, reinterpret_cast<PyObject*>(py_descriptor))); + // Ensures that the DescriptorPool stays alive. + PyDescriptorPool* pool = GetDescriptorPool_FromPool( + GetFileDescriptor(descriptor)->pool()); + if (pool == NULL) { + Py_DECREF(py_descriptor); + return NULL; + } + Py_INCREF(pool); + py_descriptor->pool = pool; + if (was_created) { *was_created = true; } @@ -295,7 +342,8 @@ PyObject* NewInternedDescriptor(PyTypeObject* type, const void* descriptor, static void Dealloc(PyBaseDescriptor* self) { // Remove from interned dictionary - GetDescriptorPool()->interned_descriptors->erase(self->descriptor); + interned_descriptors.erase(self->descriptor); + Py_CLEAR(self->pool); Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self)); } diff --git a/python/google/protobuf/pyext/descriptor_pool.cc b/python/google/protobuf/pyext/descriptor_pool.cc index d5ba2b6f..7aed651d 100644 --- a/python/google/protobuf/pyext/descriptor_pool.cc +++ b/python/google/protobuf/pyext/descriptor_pool.cc @@ -54,9 +54,13 @@ namespace google { namespace protobuf { namespace python { +// A map to cache Python Pools per C++ pointer. +// Pointers are not owned here, and belong to the PyDescriptorPool. +static hash_map<const DescriptorPool*, PyDescriptorPool*> descriptor_pool_map; + namespace cdescriptor_pool { -PyDescriptorPool* NewDescriptorPool() { +static PyDescriptorPool* NewDescriptorPool() { PyDescriptorPool* cdescriptor_pool = PyObject_New( PyDescriptorPool, &PyDescriptorPool_Type); if (cdescriptor_pool == NULL) { @@ -77,22 +81,27 @@ PyDescriptorPool* NewDescriptorPool() { // storage. cdescriptor_pool->classes_by_descriptor = new PyDescriptorPool::ClassesByMessageMap(); - cdescriptor_pool->interned_descriptors = - new hash_map<const void*, PyObject *>(); cdescriptor_pool->descriptor_options = new hash_map<const void*, PyObject *>(); + if (!descriptor_pool_map.insert( + std::make_pair(cdescriptor_pool->pool, cdescriptor_pool)).second) { + // Should never happen -- would indicate an internal error / bug. + PyErr_SetString(PyExc_ValueError, "DescriptorPool already registered"); + return NULL; + } + return cdescriptor_pool; } static void Dealloc(PyDescriptorPool* self) { typedef PyDescriptorPool::ClassesByMessageMap::iterator iterator; + descriptor_pool_map.erase(self->pool); for (iterator it = self->classes_by_descriptor->begin(); it != self->classes_by_descriptor->end(); ++it) { Py_DECREF(it->second); } delete self->classes_by_descriptor; - delete self->interned_descriptors; // its references were borrowed. for (hash_map<const void*, PyObject*>::iterator it = self->descriptor_options->begin(); it != self->descriptor_options->end(); ++it) { @@ -391,22 +400,43 @@ PyTypeObject PyDescriptorPool_Type = { PyObject_Del, // tp_free }; -static PyDescriptorPool* global_cdescriptor_pool = NULL; +// This is the DescriptorPool which contains all the definitions from the +// generated _pb2.py modules. +static PyDescriptorPool* python_generated_pool = NULL; bool InitDescriptorPool() { if (PyType_Ready(&PyDescriptorPool_Type) < 0) return false; - global_cdescriptor_pool = cdescriptor_pool::NewDescriptorPool(); - if (global_cdescriptor_pool == NULL) { + python_generated_pool = cdescriptor_pool::NewDescriptorPool(); + if (python_generated_pool == NULL) { return false; } + // Register this pool to be found for C++-generated descriptors. + descriptor_pool_map.insert( + std::make_pair(DescriptorPool::generated_pool(), + python_generated_pool)); return true; } PyDescriptorPool* GetDescriptorPool() { - return global_cdescriptor_pool; + return python_generated_pool; +} + +PyDescriptorPool* GetDescriptorPool_FromPool(const DescriptorPool* pool) { + // Fast path for standard descriptors. + if (pool == python_generated_pool->pool || + pool == DescriptorPool::generated_pool()) { + return python_generated_pool; + } + hash_map<const DescriptorPool*, PyDescriptorPool*>::iterator it = + descriptor_pool_map.find(pool); + if (it != descriptor_pool_map.end()) { + PyErr_SetString(PyExc_KeyError, "Unknown descriptor pool"); + return NULL; + } + return it->second; } } // namespace python diff --git a/python/google/protobuf/pyext/descriptor_pool.h b/python/google/protobuf/pyext/descriptor_pool.h index 6f6c5cdb..541d920b 100644 --- a/python/google/protobuf/pyext/descriptor_pool.h +++ b/python/google/protobuf/pyext/descriptor_pool.h @@ -72,14 +72,6 @@ typedef struct PyDescriptorPool { typedef hash_map<const Descriptor*, PyObject*> ClassesByMessageMap; ClassesByMessageMap* classes_by_descriptor; - // Store interned descriptors, so that the same C++ descriptor yields the same - // Python object. Objects are not immortal: this map does not own the - // references, and items are deleted when the last reference to the object is - // released. - // This is enough to support the "is" operator on live objects. - // All descriptors are stored here. - hash_map<const void*, PyObject*>* interned_descriptors; - // Cache the options for any kind of descriptor. // Descriptor pointers are owned by the DescriptorPool above. // Python objects are owned by the map. @@ -91,9 +83,6 @@ extern PyTypeObject PyDescriptorPool_Type; namespace cdescriptor_pool { -// Builds a new DescriptorPool. Normally called only once per process. -PyDescriptorPool* NewDescriptorPool(); - // Looks up a message by name. // Returns a message Descriptor, or NULL if not found. const Descriptor* FindMessageTypeByName(PyDescriptorPool* self, @@ -150,6 +139,10 @@ PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg); // Returns a *borrowed* reference. PyDescriptorPool* GetDescriptorPool(); +// Retrieve the python descriptor pool owning a C++ descriptor pool. +// Returns a *borrowed* reference. +PyDescriptorPool* GetDescriptorPool_FromPool(const DescriptorPool* pool); + // Initialize objects used by this module. bool InitDescriptorPool(); |