diff options
22 files changed, 728 insertions, 34 deletions
diff --git a/Makefile.am b/Makefile.am index 382eaa9c..6694dffe 100644 --- a/Makefile.am +++ b/Makefile.am @@ -689,6 +689,18 @@ php_EXTRA_DIST= \ php/src/Google/Protobuf/Internal/SourceCodeInfo/Location.php \ php/src/Google/Protobuf/Internal/UninterpretedOption.php \ php/src/Google/Protobuf/Internal/UninterpretedOption/NamePart.php \ + php/src/Google/Protobuf/Internal/DescriptorProto_ExtensionRange.php \ + php/src/Google/Protobuf/Internal/DescriptorProto_ReservedRange.php \ + php/src/Google/Protobuf/Internal/EnumDescriptorProto_EnumReservedRange.php \ + php/src/Google/Protobuf/Internal/FieldDescriptorProto_Label.php \ + php/src/Google/Protobuf/Internal/FieldDescriptorProto_Type.php \ + php/src/Google/Protobuf/Internal/FieldOptions_CType.php \ + php/src/Google/Protobuf/Internal/FieldOptions_JSType.php \ + php/src/Google/Protobuf/Internal/FileOptions_OptimizeMode.php \ + php/src/Google/Protobuf/Internal/GeneratedCodeInfo_Annotation.php \ + php/src/Google/Protobuf/Internal/MethodOptions_IdempotencyLevel.php \ + php/src/Google/Protobuf/Internal/SourceCodeInfo_Location.php \ + php/src/Google/Protobuf/Internal/UninterpretedOption_NamePart.php \ php/src/Google/Protobuf/ListValue.php \ php/src/Google/Protobuf/Method.php \ php/src/Google/Protobuf/Mixin.php \ @@ -804,7 +816,6 @@ python_EXTRA_DIST= \ python/google/protobuf/json_format.py \ python/google/protobuf/message.py \ python/google/protobuf/message_factory.py \ - python/google/protobuf/python_api.h \ python/google/protobuf/python_protobuf.h \ python/google/protobuf/proto_api.h \ python/google/protobuf/proto_builder.py \ @@ -825,6 +836,10 @@ python_EXTRA_DIST= \ python/google/protobuf/pyext/map_container.h \ python/google/protobuf/pyext/message.cc \ python/google/protobuf/pyext/message.h \ + python/google/protobuf/pyext/field.cc \ + python/google/protobuf/pyext/field.h \ + python/google/protobuf/pyext/unknown_fields.cc \ + python/google/protobuf/pyext/unknown_fields.h \ python/google/protobuf/pyext/message_factory.cc \ python/google/protobuf/pyext/message_factory.h \ python/google/protobuf/pyext/message_module.cc \ diff --git a/conformance/Makefile.am b/conformance/Makefile.am index 765f3588..e51ab80a 100644 --- a/conformance/Makefile.am +++ b/conformance/Makefile.am @@ -279,8 +279,6 @@ $(protoc_outputs): protoc_middleman $(other_language_protoc_outputs): protoc_middleman -BUILT_SOURCES = $(protoc_outputs) $(other_language_protoc_outputs) - CLEANFILES = $(protoc_outputs) protoc_middleman javac_middleman conformance-java javac_middleman_lite conformance-java-lite conformance-csharp conformance-php conformance-php-c $(other_language_protoc_outputs) MAINTAINERCLEANFILES = \ diff --git a/java/core/pom.xml b/java/core/pom.xml index 0c0fd8ff..9dbaff15 100644 --- a/java/core/pom.xml +++ b/java/core/pom.xml @@ -34,6 +34,11 @@ <artifactId>easymockclassextension</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> @@ -57,6 +62,15 @@ </includes> </resource> </resources> + <testResources> + <testResource> + <directory>${protobuf.source.dir}</directory> + <includes> + <include>google/protobuf/testdata/golden_message_oneof_implemented</include> + <include>google/protobuf/testdata/golden_packed_fields_message</include> + </includes> + </testResource> + </testResources> <plugins> <!-- Use Antrun plugin to generate sources with protoc --> diff --git a/java/core/src/test/proto/com/google/protobuf/map_lite_test.proto b/java/core/src/test/proto/com/google/protobuf/map_lite_test.proto index bc2105e5..c04f5d57 100644 --- a/java/core/src/test/proto/com/google/protobuf/map_lite_test.proto +++ b/java/core/src/test/proto/com/google/protobuf/map_lite_test.proto @@ -30,9 +30,10 @@ syntax = "proto3"; -package map_test; +package map_lite_test; -option java_package = "map_test"; +option optimize_for = LITE_RUNTIME; +option java_package = "map_lite_test"; option java_outer_classname = "MapTestProto"; message TestMap { diff --git a/java/core/src/test/proto/com/google/protobuf/nested_extension_lite.proto b/java/core/src/test/proto/com/google/protobuf/nested_extension_lite.proto index 6793d6b7..a95c38b2 100644 --- a/java/core/src/test/proto/com/google/protobuf/nested_extension_lite.proto +++ b/java/core/src/test/proto/com/google/protobuf/nested_extension_lite.proto @@ -38,6 +38,7 @@ syntax = "proto2"; package protobuf_unittest; +option optimize_for = LITE_RUNTIME; import "com/google/protobuf/non_nested_extension_lite.proto"; diff --git a/java/core/src/test/proto/com/google/protobuf/non_nested_extension_lite.proto b/java/core/src/test/proto/com/google/protobuf/non_nested_extension_lite.proto index e42faaa8..37c369ed 100644 --- a/java/core/src/test/proto/com/google/protobuf/non_nested_extension_lite.proto +++ b/java/core/src/test/proto/com/google/protobuf/non_nested_extension_lite.proto @@ -36,6 +36,7 @@ syntax = "proto2"; package protobuf_unittest; +option optimize_for = LITE_RUNTIME; message MessageLiteToBeExtended { extensions 1 to max; diff --git a/python/google/protobuf/pyext/extension_dict.h b/python/google/protobuf/pyext/extension_dict.h index d800d479..a7d6bb7b 100644 --- a/python/google/protobuf/pyext/extension_dict.h +++ b/python/google/protobuf/pyext/extension_dict.h @@ -37,7 +37,6 @@ #include <Python.h> #include <memory> -#include <hash_map> #include <google/protobuf/pyext/message.h> diff --git a/python/google/protobuf/pyext/field.cc b/python/google/protobuf/pyext/field.cc new file mode 100755 index 00000000..1afd4583 --- /dev/null +++ b/python/google/protobuf/pyext/field.cc @@ -0,0 +1,142 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <google/protobuf/pyext/field.h> + +#include <google/protobuf/descriptor.h> +#include <google/protobuf/pyext/descriptor.h> +#include <google/protobuf/pyext/message.h> + +#if PY_MAJOR_VERSION >= 3 + #define PyString_FromFormat PyUnicode_FromFormat +#endif + +namespace google { +namespace protobuf { +namespace python { + +namespace field { + +static PyObject* Repr(PyMessageFieldProperty* self) { + return PyString_FromFormat("<field property '%s'>", + self->field_descriptor->full_name().c_str()); +} + +static PyObject* DescrGet(PyMessageFieldProperty* self, PyObject* obj, + PyObject* type) { + if (obj == NULL) { + Py_INCREF(self); + return reinterpret_cast<PyObject*>(self); + } + return cmessage::GetFieldValue(reinterpret_cast<CMessage*>(obj), + self->field_descriptor); +} + +static int DescrSet(PyMessageFieldProperty* self, PyObject* obj, + PyObject* value) { + if (value == NULL) { + PyErr_SetString(PyExc_AttributeError, "Cannot delete field attribute"); + return -1; + } + return cmessage::SetFieldValue(reinterpret_cast<CMessage*>(obj), + self->field_descriptor, value); +} + +static PyObject* GetDescriptor(PyMessageFieldProperty* self, void* closure) { + return PyFieldDescriptor_FromDescriptor(self->field_descriptor); +} + +static PyObject* GetDoc(PyMessageFieldProperty* self, void* closure) { + return PyString_FromFormat("Field %s", + self->field_descriptor->full_name().c_str()); +} + +static PyGetSetDef Getters[] = { + {"DESCRIPTOR", (getter)GetDescriptor, NULL, "Field descriptor"}, + {"__doc__", (getter)GetDoc, NULL, NULL}, + {NULL}}; +} // namespace field + +static PyTypeObject _CFieldProperty_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) // head + FULL_MODULE_NAME ".FieldProperty", // tp_name + sizeof(PyMessageFieldProperty), // tp_basicsize + 0, // tp_itemsize + 0, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + (reprfunc)field::Repr, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "Field property of a Message", // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + 0, // tp_methods + 0, // tp_members + field::Getters, // tp_getset + 0, // tp_base + 0, // tp_dict + (descrgetfunc)field::DescrGet, // tp_descr_get + (descrsetfunc)field::DescrSet, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init + 0, // tp_alloc + 0, // tp_new +}; +PyTypeObject* CFieldProperty_Type = &_CFieldProperty_Type; + +PyObject* NewFieldProperty(const FieldDescriptor* field_descriptor) { + // Create a new descriptor object + PyMessageFieldProperty* property = + PyObject_New(PyMessageFieldProperty, CFieldProperty_Type); + if (property == NULL) { + return NULL; + } + property->field_descriptor = field_descriptor; + return reinterpret_cast<PyObject*>(property); +} + +} // namespace python +} // namespace protobuf +} // namespace google diff --git a/python/google/protobuf/pyext/field.h b/python/google/protobuf/pyext/field.h new file mode 100755 index 00000000..7b4660ca --- /dev/null +++ b/python/google/protobuf/pyext/field.h @@ -0,0 +1,59 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_FIELD_H__ +#define GOOGLE_PROTOBUF_PYTHON_CPP_FIELD_H__ + +#include <Python.h> + +namespace google { +namespace protobuf { + +class FieldDescriptor; + +namespace python { + +// A data descriptor that represents a field in a Message class. +struct PyMessageFieldProperty { + PyObject_HEAD; + + // This pointer is owned by the same pool as the Message class it belongs to. + const FieldDescriptor* field_descriptor; +}; + +extern PyTypeObject* CFieldProperty_Type; + +PyObject* NewFieldProperty(const FieldDescriptor* field_descriptor); + +} // namespace python +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_PYTHON_CPP_FIELD_H__ diff --git a/python/google/protobuf/pyext/map_container.cc b/python/google/protobuf/pyext/map_container.cc index 3eec49c7..cc57d1c2 100644 --- a/python/google/protobuf/pyext/map_container.cc +++ b/python/google/protobuf/pyext/map_container.cc @@ -75,7 +75,7 @@ class MapReflectionFriend { struct MapIterator { PyObject_HEAD; - std::unique_ptr<::proto2::MapIterator> iter; + std::unique_ptr<google::protobuf::MapIterator> iter; // A pointer back to the container, so we can notice changes to the version. // We own a ref on this. @@ -314,7 +314,7 @@ static MapContainer* GetMap(PyObject* obj) { Py_ssize_t MapReflectionFriend::Length(PyObject* _self) { MapContainer* self = GetMap(_self); - const proto2::Message* message = self->message; + const Message* message = self->message; return message->GetReflection()->MapSize(*message, self->parent_field_descriptor); } @@ -423,7 +423,7 @@ int MapContainer::Release() { // ScalarMap /////////////////////////////////////////////////////////////////// PyObject *NewScalarMapContainer( - CMessage* parent, const proto2::FieldDescriptor* parent_field_descriptor) { + CMessage* parent, const FieldDescriptor* parent_field_descriptor) { if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) { return NULL; } @@ -548,7 +548,7 @@ PyObject* MapReflectionFriend::ScalarMapToStr(PyObject* _self) { MapContainer* self = GetMap(_self); Message* message = self->GetMutableMessage(); const Reflection* reflection = message->GetReflection(); - for (proto2::MapIterator it = reflection->MapBegin( + for (google::protobuf::MapIterator it = reflection->MapBegin( message, self->parent_field_descriptor); it != reflection->MapEnd(message, self->parent_field_descriptor); ++it) { @@ -704,7 +704,7 @@ static PyObject* GetCMessage(MessageMapContainer* self, Message* message, } PyObject* NewMessageMapContainer( - CMessage* parent, const proto2::FieldDescriptor* parent_field_descriptor, + CMessage* parent, const FieldDescriptor* parent_field_descriptor, CMessageClass* message_class) { if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) { return NULL; @@ -837,7 +837,7 @@ PyObject* MapReflectionFriend::MessageMapToStr(PyObject* _self) { MessageMapContainer* self = GetMessageMap(_self); Message* message = self->GetMutableMessage(); const Reflection* reflection = message->GetReflection(); - for (proto2::MapIterator it = reflection->MapBegin( + for (google::protobuf::MapIterator it = reflection->MapBegin( message, self->parent_field_descriptor); it != reflection->MapEnd(message, self->parent_field_descriptor); ++it) { @@ -1011,7 +1011,7 @@ PyObject* MapReflectionFriend::GetIterator(PyObject *_self) { Message* message = self->GetMutableMessage(); const Reflection* reflection = message->GetReflection(); - iter->iter.reset(new ::proto2::MapIterator( + iter->iter.reset(new google::protobuf::MapIterator( reflection->MapBegin(message, self->parent_field_descriptor))); } diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc index 5d0e37fa..e138c93d 100644 --- a/python/google/protobuf/pyext/message.cc +++ b/python/google/protobuf/pyext/message.cc @@ -171,7 +171,7 @@ static int AddDescriptors(PyObject* cls, const Descriptor* descriptor) { // <message descriptor>.extensions_by_name[name] // which was defined previously. for (int i = 0; i < descriptor->extension_count(); ++i) { - const proto2::FieldDescriptor* field = descriptor->extension(i); + const FieldDescriptor* field = descriptor->extension(i); ScopedPyObjectPtr extension_field(PyFieldDescriptor_FromDescriptor(field)); if (extension_field == NULL) { return -1; @@ -2344,7 +2344,7 @@ static PyObject* RichCompare(CMessage* self, PyObject* other, int opid) { if (!PyObject_TypeCheck(other, CMessage_Type)) { equals = false; } - const proto2::Message* other_message = + const Message* other_message = reinterpret_cast<CMessage*>(other)->message; // If messages don't have the same descriptors, they are not equal. if (equals && @@ -2352,7 +2352,7 @@ static PyObject* RichCompare(CMessage* self, PyObject* other, int opid) { equals = false; } // Check the message contents. - if (equals && !proto2::util::MessageDifferencer::Equals( + if (equals && !util::MessageDifferencer::Equals( *self->message, *reinterpret_cast<CMessage*>(other)->message)) { equals = false; diff --git a/python/google/protobuf/pyext/message.h b/python/google/protobuf/pyext/message.h index e729e448..cbd422be 100644 --- a/python/google/protobuf/pyext/message.h +++ b/python/google/protobuf/pyext/message.h @@ -38,7 +38,7 @@ #include <memory> #include <string> -#include <hash_map> +#include <unordered_map> #include <google/protobuf/stubs/common.h> #include <google/protobuf/pyext/thread_unsafe_shared_ptr.h> @@ -105,7 +105,7 @@ typedef struct CMessage { // Also cache extension fields. // The FieldDescriptor is owned by the message's pool; PyObject references // are owned. - typedef __gnu_cxx::hash_map<const FieldDescriptor*, PyObject*> + typedef std::unordered_map<const FieldDescriptor*, PyObject*> CompositeFieldsMap; CompositeFieldsMap* composite_fields; diff --git a/python/google/protobuf/pyext/message_module.cc b/python/google/protobuf/pyext/message_module.cc index 29d56702..8d465eb5 100644 --- a/python/google/protobuf/pyext/message_module.cc +++ b/python/google/protobuf/pyext/message_module.cc @@ -31,19 +31,19 @@ #include <Python.h> #include <google/protobuf/pyext/message.h> -#include <google/protobuf/python/proto_api.h> +#include <google/protobuf/proto_api.h> #include <google/protobuf/message_lite.h> namespace { // C++ API. Clients get at this via proto_api.h -struct ApiImplementation : proto2::python::PyProto_API { - const proto2::Message* GetMessagePointer(PyObject* msg) const override { - return proto2::python::PyMessage_GetMessagePointer(msg); +struct ApiImplementation : google::protobuf::python::PyProto_API { + const google::protobuf::Message* GetMessagePointer(PyObject* msg) const override { + return google::protobuf::python::PyMessage_GetMessagePointer(msg); } - proto2::Message* GetMutableMessagePointer(PyObject* msg) const override { - return proto2::python::PyMessage_GetMutableMessagePointer(msg); + google::protobuf::Message* GetMutableMessagePointer(PyObject* msg) const override { + return google::protobuf::python::PyMessage_GetMutableMessagePointer(msg); } }; @@ -58,7 +58,7 @@ static const char module_docstring[] = static PyMethodDef ModuleMethods[] = { {"SetAllowOversizeProtos", - (PyCFunction)proto2::python::cmessage::SetAllowOversizeProtos, + (PyCFunction)google::protobuf::python::cmessage::SetAllowOversizeProtos, METH_O, "Enable/disable oversize proto parsing."}, // DO NOT USE: For migration and testing only. { NULL, NULL} @@ -94,7 +94,7 @@ PyMODINIT_FUNC INITFUNC() { return INITFUNC_ERRORVAL; } - if (!proto2::python::InitProto2MessageModule(m)) { + if (!google::protobuf::python::InitProto2MessageModule(m)) { Py_DECREF(m); return INITFUNC_ERRORVAL; } @@ -102,7 +102,7 @@ PyMODINIT_FUNC INITFUNC() { // Adds the C++ API if (PyObject* api = PyCapsule_New(new ApiImplementation(), - proto2::python::PyProtoAPICapsuleName(), NULL)) { + google::protobuf::python::PyProtoAPICapsuleName(), NULL)) { PyModule_AddObject(m, "proto_API", api); } else { return INITFUNC_ERRORVAL; diff --git a/python/google/protobuf/pyext/unknown_fields.cc b/python/google/protobuf/pyext/unknown_fields.cc new file mode 100755 index 00000000..760452f2 --- /dev/null +++ b/python/google/protobuf/pyext/unknown_fields.cc @@ -0,0 +1,355 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <google/protobuf/pyext/unknown_fields.h> + +#include <Python.h> +#include <set> +#include <memory> + +#include <google/protobuf/message.h> +#include <google/protobuf/pyext/message.h> +#include <google/protobuf/pyext/scoped_pyobject_ptr.h> +#include <google/protobuf/unknown_field_set.h> +#include <google/protobuf/wire_format_lite.h> + +#if PY_MAJOR_VERSION >= 3 + #define PyInt_FromLong PyLong_FromLong +#endif + +namespace google { +namespace protobuf { +namespace python { + +namespace unknown_fields { + +static Py_ssize_t Len(PyObject* pself) { + PyUnknownFields* self = + reinterpret_cast<PyUnknownFields*>(pself); + if (self->fields == NULL) { + PyErr_Format(PyExc_ValueError, + "UnknownFields does not exist. " + "The parent message might be cleared."); + return -1; + } + return self->fields->field_count(); +} + +void Clear(PyUnknownFields* self) { + for (std::set<PyUnknownFields*>::iterator it = + self->sub_unknown_fields.begin(); + it != self->sub_unknown_fields.end(); it++) { + Clear(*it); + } + self->fields = NULL; + self->sub_unknown_fields.clear(); +} + +PyObject* NewPyUnknownFieldRef(PyUnknownFields* parent, + Py_ssize_t index); + +static PyObject* Item(PyObject* pself, Py_ssize_t index) { + PyUnknownFields* self = + reinterpret_cast<PyUnknownFields*>(pself); + if (self->fields == NULL) { + PyErr_Format(PyExc_ValueError, + "UnknownFields does not exist. " + "The parent message might be cleared."); + return NULL; + } + Py_ssize_t total_size = self->fields->field_count(); + if (index < 0) { + index = total_size + index; + } + if (index < 0 || index >= total_size) { + PyErr_Format(PyExc_IndexError, + "index (%zd) out of range", + index); + return NULL; + } + + return unknown_fields::NewPyUnknownFieldRef(self, index); +} + +PyObject* NewPyUnknownFields(CMessage* c_message) { + PyUnknownFields* self = reinterpret_cast<PyUnknownFields*>( + PyType_GenericAlloc(&PyUnknownFields_Type, 0)); + if (self == NULL) { + return NULL; + } + // Call "placement new" to initialize PyUnknownFields. + new (self) PyUnknownFields; + + Py_INCREF(c_message); + self->parent = reinterpret_cast<PyObject*>(c_message); + Message* message = c_message->message; + const Reflection* reflection = message->GetReflection(); + self->fields = &reflection->GetUnknownFields(*message); + + return reinterpret_cast<PyObject*>(self); +} + +PyObject* NewPyUnknownFieldRef(PyUnknownFields* parent, + Py_ssize_t index) { + PyUnknownFieldRef* self = reinterpret_cast<PyUnknownFieldRef*>( + PyType_GenericAlloc(&PyUnknownFieldRef_Type, 0)); + if (self == NULL) { + return NULL; + } + + Py_INCREF(parent); + self->parent = parent; + self->index = index; + + return reinterpret_cast<PyObject*>(self); +} + +static void Dealloc(PyObject* pself) { + PyUnknownFields* self = + reinterpret_cast<PyUnknownFields*>(pself); + if (PyObject_TypeCheck(self->parent, &PyUnknownFields_Type)) { + reinterpret_cast<PyUnknownFields*>( + self->parent)->sub_unknown_fields.erase(self); + } + Py_CLEAR(self->parent); + self->~PyUnknownFields(); +} + +static PySequenceMethods SqMethods = { + Len, /* sq_length */ + 0, /* sq_concat */ + 0, /* sq_repeat */ + Item, /* sq_item */ + 0, /* sq_slice */ + 0, /* sq_ass_item */ +}; + +} // namespace unknown_fields + +PyTypeObject PyUnknownFields_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + FULL_MODULE_NAME ".PyUnknownFields", // tp_name + sizeof(PyUnknownFields), // tp_basicsize + 0, // tp_itemsize + unknown_fields::Dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + &unknown_fields::SqMethods, // tp_as_sequence + 0, // tp_as_mapping + PyObject_HashNotImplemented, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "unknown field set", // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + 0, // tp_methods + 0, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init +}; + +namespace unknown_field { +static PyObject* PyUnknownFields_FromUnknownFieldSet( + PyUnknownFields* parent, const UnknownFieldSet& fields) { + PyUnknownFields* self = reinterpret_cast<PyUnknownFields*>( + PyType_GenericAlloc(&PyUnknownFields_Type, 0)); + if (self == NULL) { + return NULL; + } + // Call "placement new" to initialize PyUnknownFields. + new (self) PyUnknownFields; + + Py_INCREF(parent); + self->parent = reinterpret_cast<PyObject*>(parent); + self->fields = &fields; + parent->sub_unknown_fields.emplace(self); + + return reinterpret_cast<PyObject*>(self); +} + +const UnknownField* GetUnknownField(PyUnknownFieldRef* self) { + const UnknownFieldSet* fields = self->parent->fields; + if (fields == NULL) { + PyErr_Format(PyExc_ValueError, + "UnknownField does not exist. " + "The parent message might be cleared."); + return NULL; + } + ssize_t total_size = fields->field_count(); + if (self->index >= total_size) { + PyErr_Format(PyExc_ValueError, + "UnknownField does not exist. " + "The parent message might be cleared."); + return NULL; + } + return &fields->field(self->index); +} + +static PyObject* GetFieldNumber(PyUnknownFieldRef* self, void *closure) { + const UnknownField* unknown_field = GetUnknownField(self); + if (unknown_field == NULL) { + return NULL; + } + return PyInt_FromLong(unknown_field->number()); +} + +using internal::WireFormatLite; +static PyObject* GetWireType(PyUnknownFieldRef* self, void *closure) { + const UnknownField* unknown_field = GetUnknownField(self); + if (unknown_field == NULL) { + return NULL; + } + + // Assign a default value to suppress may-unintialized warnings (errors + // when built in some places). + WireFormatLite::WireType wire_type = WireFormatLite::WIRETYPE_VARINT; + switch (unknown_field->type()) { + case UnknownField::TYPE_VARINT: + wire_type = WireFormatLite::WIRETYPE_VARINT; + break; + case UnknownField::TYPE_FIXED32: + wire_type = WireFormatLite::WIRETYPE_FIXED32; + break; + case UnknownField::TYPE_FIXED64: + wire_type = WireFormatLite::WIRETYPE_FIXED64; + break; + case UnknownField::TYPE_LENGTH_DELIMITED: + wire_type = WireFormatLite::WIRETYPE_LENGTH_DELIMITED; + break; + case UnknownField::TYPE_GROUP: + wire_type = WireFormatLite::WIRETYPE_START_GROUP; + break; + } + return PyInt_FromLong(wire_type); +} + +static PyObject* GetData(PyUnknownFieldRef* self, void *closure) { + const UnknownField* field = GetUnknownField(self); + if (field == NULL) { + return NULL; + } + PyObject* data = NULL; + switch (field->type()) { + case UnknownField::TYPE_VARINT: + data = PyInt_FromLong(field->varint()); + break; + case UnknownField::TYPE_FIXED32: + data = PyInt_FromLong(field->fixed32()); + break; + case UnknownField::TYPE_FIXED64: + data = PyInt_FromLong(field->fixed64()); + break; + case UnknownField::TYPE_LENGTH_DELIMITED: + data = PyBytes_FromStringAndSize(field->length_delimited().data(), + field->GetLengthDelimitedSize()); + break; + case UnknownField::TYPE_GROUP: + data = PyUnknownFields_FromUnknownFieldSet( + self->parent, field->group()); + break; + } + return data; +} + +static void Dealloc(PyObject* pself) { + PyUnknownFieldRef* self = + reinterpret_cast<PyUnknownFieldRef*>(pself); + Py_CLEAR(self->parent); +} + +static PyGetSetDef Getters[] = { + {"field_number", (getter)GetFieldNumber, NULL}, + {"wire_type", (getter)GetWireType, NULL}, + {"data", (getter)GetData, NULL}, + {NULL} +}; + +} // namespace unknown_field + +PyTypeObject PyUnknownFieldRef_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + FULL_MODULE_NAME ".PyUnknownFieldRef", // tp_name + sizeof(PyUnknownFieldRef), // tp_basicsize + 0, // tp_itemsize + unknown_field::Dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + PyObject_HashNotImplemented, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "unknown field", // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + 0, // tp_methods + 0, // tp_members + unknown_field::Getters, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init +}; + + +} // namespace python +} // namespace protobuf +} // namespace google diff --git a/python/google/protobuf/pyext/unknown_fields.h b/python/google/protobuf/pyext/unknown_fields.h new file mode 100755 index 00000000..94d55e14 --- /dev/null +++ b/python/google/protobuf/pyext/unknown_fields.h @@ -0,0 +1,90 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_UNKNOWN_FIELDS_H__ +#define GOOGLE_PROTOBUF_PYTHON_CPP_UNKNOWN_FIELDS_H__ + +#include <Python.h> + +#include <memory> +#include <set> + +#include <google/protobuf/pyext/message.h> + +namespace google { +namespace protobuf { + +class UnknownField; +class UnknownFieldSet; + +namespace python { +struct CMessage; + +typedef struct PyUnknownFields { + PyObject_HEAD; + // Strong pointer to the parent CMessage or PyUnknownFields. + // The top PyUnknownFields holds a reference to its parent CMessage + // object before release. + // Sub PyUnknownFields holds reference to parent PyUnknownFields. + PyObject* parent; + + // Pointer to the C++ UnknownFieldSet. + // PyUnknownFields does not own this pointer. + const UnknownFieldSet* fields; + + // Weak references to child unknown fields. + std::set<PyUnknownFields*> sub_unknown_fields; +} PyUnknownFields; + +typedef struct PyUnknownFieldRef { + PyObject_HEAD; + // Every Python PyUnknownFieldRef holds a reference to its parent + // PyUnknownFields in order to keep it alive. + PyUnknownFields* parent; + + // The UnknownField index in UnknownFields. + Py_ssize_t index; +} UknownFieldRef; + +extern PyTypeObject PyUnknownFields_Type; +extern PyTypeObject PyUnknownFieldRef_Type; + +namespace unknown_fields { + +// Builds an PyUnknownFields for a specific message. +PyObject* NewPyUnknownFields(CMessage *parent); +void Clear(PyUnknownFields* self); + +} // namespace unknown_fields +} // namespace python +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_PYTHON_CPP_UNKNOWN_FIELDS_H__ diff --git a/python/setup.py b/python/setup.py index 5ec4dcbd..63c2d0e0 100755 --- a/python/setup.py +++ b/python/setup.py @@ -185,8 +185,12 @@ if __name__ == '__main__': extra_compile_args.append('-Wno-write-strings') extra_compile_args.append('-Wno-invalid-offsetof') extra_compile_args.append('-Wno-sign-compare') + extra_compile_args.append('-Wno-unused-variable') extra_compile_args.append('-std=c++11') + if sys.platform == 'darwin': + extra_compile_args.append("-Wno-shorten-64-to-32"); + # https://github.com/Theano/Theano/issues/4926 if sys.platform == 'win32': extra_compile_args.append('-D_hypot=hypot') diff --git a/src/Makefile.am b/src/Makefile.am index af264a95..1aabf972 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -112,6 +112,9 @@ nobase_include_HEADERS = \ google/protobuf/message_lite.h \ google/protobuf/metadata.h \ google/protobuf/metadata_lite.h \ + google/protobuf/port.h \ + google/protobuf/port_def.inc \ + google/protobuf/port_undef.inc \ google/protobuf/reflection.h \ google/protobuf/reflection_ops.h \ google/protobuf/repeated_field.h \ @@ -690,6 +693,7 @@ COMMON_TEST_SOURCES = \ google/protobuf/test_util.cc \ google/protobuf/test_util.h \ google/protobuf/test_util.inc \ + google/protobuf/test_util2.h \ google/protobuf/testing/googletest.cc \ google/protobuf/testing/googletest.h \ google/protobuf/testing/file.cc \ @@ -743,6 +747,7 @@ protobuf_test_SOURCES = \ google/protobuf/proto3_arena_lite_unittest.cc \ google/protobuf/proto3_arena_unittest.cc \ google/protobuf/proto3_lite_unittest.cc \ + google/protobuf/proto3_lite_unittest.inc \ google/protobuf/reflection_ops_unittest.cc \ google/protobuf/repeated_field_reflection_unittest.cc \ google/protobuf/repeated_field_unittest.cc \ diff --git a/src/google/protobuf/compiler/java/java_message_lite.cc b/src/google/protobuf/compiler/java/java_message_lite.cc index d2cc5f95..85a7453d 100644 --- a/src/google/protobuf/compiler/java/java_message_lite.cc +++ b/src/google/protobuf/compiler/java/java_message_lite.cc @@ -737,10 +737,10 @@ void ImmutableMessageLiteGenerator::GenerateSerializeOneExtensionRange( void ImmutableMessageLiteGenerator::GenerateBuilder(io::Printer* printer) { printer->Print( "public static Builder newBuilder() {\n" - " return DEFAULT_INSTANCE.createBuilder();\n" + " return (Builder) DEFAULT_INSTANCE.createBuilder();\n" "}\n" "public static Builder newBuilder($classname$ prototype) {\n" - " return DEFAULT_INSTANCE.createBuilder(prototype);\n" + " return (Builder) DEFAULT_INSTANCE.createBuilder(prototype);\n" "}\n" "\n", "classname", name_resolver_->GetImmutableClassName(descriptor_)); diff --git a/src/google/protobuf/descriptor_database.h b/src/google/protobuf/descriptor_database.h index 15ed8c64..0a87a147 100644 --- a/src/google/protobuf/descriptor_database.h +++ b/src/google/protobuf/descriptor_database.h @@ -164,7 +164,7 @@ class LIBPROTOBUF_EXPORT SimpleDescriptorDatabase : public DescriptorDatabase { bool FindAllExtensionNumbers(const string& extendee_type, std::vector<int>* output) override; - bool FindAllFileNames(std::vector<string>* output); + bool FindAllFileNames(std::vector<string>* output) override; private: // So that it can use DescriptorIndex. diff --git a/src/google/protobuf/stubs/strutil.h b/src/google/protobuf/stubs/strutil.h index d6bcbc32..ec5512d6 100644 --- a/src/google/protobuf/stubs/strutil.h +++ b/src/google/protobuf/stubs/strutil.h @@ -913,6 +913,15 @@ LIBPROTOBUF_EXPORT void CleanStringLineEndings(const string& src, string* dst, LIBPROTOBUF_EXPORT void CleanStringLineEndings(string* str, bool auto_end_last_line); +namespace strings { +inline bool EndsWith(StringPiece text, StringPiece suffix) { + return suffix.empty() || + (text.size() >= suffix.size() && + memcmp(text.data() + (text.size() - suffix.size()), suffix.data(), + suffix.size()) == 0); +} +} // namespace strings + } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/text_format.cc b/src/google/protobuf/text_format.cc index 09267be1..93c24b23 100644 --- a/src/google/protobuf/text_format.cc +++ b/src/google/protobuf/text_format.cc @@ -1356,8 +1356,9 @@ bool CheckParseInputSize(StringPiece input, io::ErrorCollector* error_collector) { if (input.size() > INT_MAX) { error_collector->AddError( - -1, 0, StrCat("Input size too large: ", input.size(), " bytes", - " > ", INT_MAX, " bytes.")); + -1, 0, StrCat("Input size too large: ", + static_cast<int64>(input.size()), " bytes", + " > ", INT_MAX, " bytes.")); return false; } return true; @@ -56,10 +56,10 @@ build_cpp_distcheck() { # Check if every file exists in the dist tar file. FILES_MISSING="" for FILE in $(<../dist.lst); do - if ! file $FILE &>/dev/null; then + [ -f "$FILE" ] || { echo "$FILE is not found!" FILES_MISSING="$FILE $FILES_MISSING" - fi + } done cd .. if [ ! -z "$FILES_MISSING" ]; then |