diff options
118 files changed, 12660 insertions, 3235 deletions
@@ -16,6 +16,7 @@ m4/ltoptions.m4 m4/ltsugar.m4 m4/ltversion.m4 m4/lt~obsolete.m4 +autom4te.cache # downloaded files gtest @@ -23,7 +24,7 @@ gtest # in-tree configure-generated files Makefile src/Makefile -config.h +/config.h config.log config.status @@ -61,3 +62,13 @@ src/test_plugin src/testzip.* src/zcg*zip ar-lib + +test-driver +compile + +src/**/*.log +src/**/*.trs + +# JavaBuild output. +java/target +javanano/target diff --git a/CHANGES.txt b/CHANGES.txt index 0d4ce0ec..ac42aa5a 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,85 @@ +2015-02-26 version 3.0.0-alpha-2 (Python/Ruby/JavaNano): + General + * Introduced three new language implementations (Ruby, JavaNano, and + Python) to proto3. + * Various bug fixes since 3.0.0-alpha-1 + + Python: + Python has received several updates, most notably support for proto3 + semantics in any .proto file that declares syntax="proto3". + Messages declared in proto3 files no longer represent field presence + for scalar fields (number, enums, booleans, or strings). You can + no longer call HasField() for such fields, and they are serialized + based on whether they have a non-zero/empty/false value. + + One other notable change is in the C++-accelerated implementation. + Descriptor objects (which describe the protobuf schema and allow + reflection over it) are no longer duplicated between the Python + and C++ layers. The Python descriptors are now simple wrappers + around the C++ descriptors. This change should significantly + reduce the memory usage of programs that use a lot of message + types. + + Ruby: + We have added proto3 support for Ruby via a native C extension. + + The Ruby extension itself is included in the ruby/ directory, and details on + building and installing the extension are in ruby/README.md. The extension + will also be published as a Ruby gem. Code generator support is included as + part of `protoc` with the `--ruby_out` flag. + + The Ruby extension implements a user-friendly DSL to define message types + (also generated by the code generator from `.proto` files). Once a message + type is defined, the user may create instances of the message that behave in + ways idiomatic to Ruby. For example: + + - Message fields are present as ordinary Ruby properties (getter method + `foo` and setter method `foo=`). + - Repeated field elements are stored in a container that acts like a native + Ruby array, and map elements are stored in a container that acts like a + native Ruby hashmap. + - The usual well-known methods, such as `#to_s`, `#dup`, and the like, are + present. + + Unlike several existing third-party Ruby extensions for protobuf, this + extension is built on a "strongly-typed" philosophy: message fields and + array/map containers will throw exceptions eagerly when values of the + incorrect type are inserted. + + See ruby/README.md for details. + + JavaNano: + JavaNano is a special code generator and runtime library designed especially + for resource-restricted systems, like Android. It is very resource-friendly + in both the amount of code and the runtime overhead. Here is an an overview + of JavaNano features compared with the official Java protobuf: + + - No descriptors or message builders. + - All messages are mutable; fields are public Java fields. + - For optional fields only, encapsulation behind setter/getter/hazzer/ + clearer functions is opt-in, which provide proper 'has' state support. + - For proto2, if not opted in, has state (field presence) is not available. + Serialization outputs all fields not equal to their defaults. + The behavior is consistent with proto3 semantics. + - Required fields (proto2 only) are always serialized. + - Enum constants are integers; protection against invalid values only + when parsing from the wire. + - Enum constants can be generated into container interfaces bearing + the enum's name (so the referencing code is in Java style). + - CodedInputByteBufferNano can only take byte[] (not InputStream). + - Similarly CodedOutputByteBufferNano can only write to byte[]. + - Repeated fields are in arrays, not ArrayList or Vector. Null array + elements are allowed and silently ignored. + - Full support for serializing/deserializing repeated packed fields. + - Support extensions (in proto2). + - Unset messages/groups are null, not an immutable empty default + instance. + - toByteArray(...) and mergeFrom(...) are now static functions of + MessageNano. + - The 'bytes' type translates to the Java type byte[]. + + See javanano/README.txt for details. + 2014-12-01 version 3.0.0-alpha-1 (C++/Java): General diff --git a/Makefile.am b/Makefile.am index a1f7e0ba..1c039230 100644 --- a/Makefile.am +++ b/Makefile.am @@ -154,10 +154,49 @@ java_EXTRA_DIST= \ java/pom.xml \ java/README.txt +javanano_EXTRA_DIST= \ + javanano/src/main/java/com/google/protobuf/nano/CodedOutputByteBufferNano.java \ + javanano/src/main/java/com/google/protobuf/nano/FieldData.java \ + javanano/src/main/java/com/google/protobuf/nano/FieldArray.java \ + javanano/src/main/java/com/google/protobuf/nano/WireFormatNano.java \ + javanano/src/main/java/com/google/protobuf/nano/Extension.java \ + javanano/src/main/java/com/google/protobuf/nano/CodedInputByteBufferNano.java \ + javanano/src/main/java/com/google/protobuf/nano/UnknownFieldData.java \ + javanano/src/main/java/com/google/protobuf/nano/MessageNano.java \ + javanano/src/main/java/com/google/protobuf/nano/InternalNano.java \ + javanano/src/main/java/com/google/protobuf/nano/InvalidProtocolBufferNanoException.java \ + javanano/src/main/java/com/google/protobuf/nano/MapFactories.java \ + javanano/src/main/java/com/google/protobuf/nano/ExtendableMessageNano.java \ + javanano/src/main/java/com/google/protobuf/nano/MessageNanoPrinter.java \ + javanano/src/test/java/com/google/protobuf/nano/unittest_accessors_nano.proto \ + javanano/src/test/java/com/google/protobuf/nano/unittest_enum_class_nano.proto \ + javanano/src/test/java/com/google/protobuf/nano/unittest_reference_types_nano.proto \ + javanano/src/test/java/com/google/protobuf/nano/unittest_extension_repeated_nano.proto \ + javanano/src/test/java/com/google/protobuf/nano/unittest_has_nano.proto \ + javanano/src/test/java/com/google/protobuf/nano/unittest_nano.proto \ + javanano/src/test/java/com/google/protobuf/nano/unittest_multiple_nameclash_nano.proto \ + javanano/src/test/java/com/google/protobuf/nano/unittest_single_nano.proto \ + javanano/src/test/java/com/google/protobuf/nano/NanoTest.java \ + javanano/src/test/java/com/google/protobuf/nano/unittest_simple_nano.proto \ + javanano/src/test/java/com/google/protobuf/nano/unittest_import_nano.proto \ + javanano/src/test/java/com/google/protobuf/nano/unittest_repeated_merge_nano.proto \ + javanano/src/test/java/com/google/protobuf/nano/unittest_extension_nano.proto \ + javanano/src/test/java/com/google/protobuf/nano/unittest_repeated_packables_nano.proto \ + javanano/src/test/java/com/google/protobuf/nano/unittest_extension_singular_nano.proto \ + javanano/src/test/java/com/google/protobuf/nano/unittest_recursive_nano.proto \ + javanano/src/test/java/com/google/protobuf/nano/unittest_extension_packed_nano.proto \ + javanano/src/test/java/com/google/protobuf/nano/unittest_enum_validity_nano.proto \ + javanano/src/test/java/com/google/protobuf/nano/unittest_stringutf8_nano.proto \ + javanano/src/test/java/com/google/protobuf/nano/unittest_multiple_nano.proto \ + javanano/src/test/java/com/google/protobuf/nano/unittest_enum_class_multiple_nano.proto \ + javanano/src/test/java/com/google/protobuf/nano/map_test.proto \ + javanano/README.txt \ + javanano/pom.xml + + python_EXTRA_DIST= \ python/google/protobuf/internal/api_implementation.cc \ python/google/protobuf/internal/api_implementation.py \ - python/google/protobuf/internal/api_implementation_default_test.py \ python/google/protobuf/internal/containers.py \ python/google/protobuf/internal/cpp_message.py \ python/google/protobuf/internal/decoder.py \ @@ -181,6 +220,7 @@ python_EXTRA_DIST= \ python/google/protobuf/internal/more_extensions.proto \ python/google/protobuf/internal/more_extensions_dynamic.proto \ python/google/protobuf/internal/more_messages.proto \ + python/google/protobuf/internal/_parameterized.py \ python/google/protobuf/internal/proto_builder_test.py \ python/google/protobuf/internal/python_message.py \ python/google/protobuf/internal/reflection_test.py \ @@ -202,16 +242,17 @@ python_EXTRA_DIST= \ python/google/protobuf/pyext/cpp_message.py \ python/google/protobuf/pyext/descriptor.h \ python/google/protobuf/pyext/descriptor.cc \ - python/google/protobuf/pyext/descriptor_cpp2_test.py \ + python/google/protobuf/pyext/descriptor_pool.h \ + python/google/protobuf/pyext/descriptor_pool.cc \ + python/google/protobuf/pyext/descriptor_containers.h \ + python/google/protobuf/pyext/descriptor_containers.cc \ python/google/protobuf/pyext/extension_dict.h \ python/google/protobuf/pyext/extension_dict.cc \ python/google/protobuf/pyext/message.h \ python/google/protobuf/pyext/message.cc \ - python/google/protobuf/pyext/message_factory_cpp2_test.py \ python/google/protobuf/pyext/proto2_api_test.proto \ python/google/protobuf/pyext/python.proto \ python/google/protobuf/pyext/python_protobuf.h \ - python/google/protobuf/pyext/reflection_cpp2_generated_test.py \ python/google/protobuf/pyext/repeated_composite_container.h \ python/google/protobuf/pyext/repeated_composite_container.cc \ python/google/protobuf/pyext/repeated_scalar_container.h \ @@ -244,6 +285,7 @@ ruby_EXTRA_DIST= \ ruby/ext/google/protobuf_c/defs.c \ ruby/ext/google/protobuf_c/encode_decode.c \ ruby/ext/google/protobuf_c/extconf.rb \ + ruby/ext/google/protobuf_c/map.c \ ruby/ext/google/protobuf_c/message.c \ ruby/ext/google/protobuf_c/protobuf.c \ ruby/ext/google/protobuf_c/protobuf.h \ @@ -254,9 +296,12 @@ ruby_EXTRA_DIST= \ ruby/google-protobuf.gemspec \ ruby/lib/google/protobuf.rb \ ruby/tests/basic.rb \ - ruby/tests/stress.rb + ruby/tests/stress.rb \ + ruby/tests/generated_code.proto \ + ruby/tests/generated_code.rb \ + ruby/tests/generated_code_test.rb -all_EXTRA_DIST=$(java_EXTRA_DIST) $(python_EXTRA_DIST) $(ruby_EXTRA_DIST) +all_EXTRA_DIST=$(java_EXTRA_DIST) $(javanano_EXTRA_DIST) $(python_EXTRA_DIST) $(ruby_EXTRA_DIST) EXTRA_DIST = $(@DIST_LANG@_EXTRA_DIST) \ autogen.sh \ @@ -266,10 +311,12 @@ EXTRA_DIST = $(@DIST_LANG@_EXTRA_DIST) \ LICENSE \ CONTRIBUTORS.txt \ CHANGES.txt \ + config.h.include \ editors/README.txt \ editors/proto.vim \ editors/protobuf-mode.el \ vsprojects/config.h \ + vsprojects/google/protobuf/stubs/pbconfig.h \ vsprojects/extract_includes.bat \ vsprojects/libprotobuf.vcproj \ vsprojects/libprotobuf-lite.vcproj \ @@ -88,6 +88,22 @@ For advanced usage information on configure and make, see INSTALL.txt. If you only want protobuf-lite, substitute "protobuf-lite" in place of "protobuf" in these examples. +**Note for Mac users** + + For a Mac system, Unix tools are not available by default. You will first need + to install Xcode from the Mac AppStore and then run the following command from + a terminal: + + $ sudo xcode-select --install + + To install Unix tools, you can install "port" following the instructions at + https://www.macports.org . This will reside in /opt/local/bin/port for most + Mac installations. + + $ sudo /opt/local/bin/port install autoconf automake libtool + + Then follow the Unix instructions above. + **Note for cross-compiling** The makefiles normally invoke the protoc executable that they just diff --git a/config.h.include b/config.h.include new file mode 100644 index 00000000..0c557fbe --- /dev/null +++ b/config.h.include @@ -0,0 +1,7 @@ +HASH_MAP_CLASS +HASH_MAP_H +HASH_NAMESPACE +HASH_SET_CLASS +HASH_SET_H +HAVE_HASH_MAP +HAVE_HASH_SET diff --git a/configure.ac b/configure.ac index d1fde9d5..b5eb9399 100644 --- a/configure.ac +++ b/configure.ac @@ -12,7 +12,7 @@ AC_PREREQ(2.59) # In the SVN trunk, the version should always be the next anticipated release # version with the "-pre" suffix. (We used to use "-SNAPSHOT" but this pushed # the size of one file name in the dist tarfile over the 99-char limit.) -AC_INIT([Protocol Buffers],[3.0.0-pre],[protobuf@googlegroups.com],[protobuf]) +AC_INIT([Protocol Buffers],[3.0.0-alpha-2],[protobuf@googlegroups.com],[protobuf]) AM_MAINTAINER_MODE([enable]) @@ -37,7 +37,7 @@ AS_IF([test "x${ac_cv_env_CXXFLAGS_set}" = "x"], AC_CANONICAL_TARGET -AM_INIT_AUTOMAKE([subdir-objects]) +AM_INIT_AUTOMAKE([1.9 tar-ustar subdir-objects]) AC_ARG_WITH([zlib], [AS_HELP_STRING([--with-zlib], @@ -54,7 +54,7 @@ AC_PROG_CC AC_PROG_CXX AC_LANG([C++]) ACX_USE_SYSTEM_EXTENSIONS -AM_PROG_AR +m4_ifdef([AM_PROG_AR], [AM_PROG_AR]) AM_CONDITIONAL(GCC, test "$GCC" = yes) # let the Makefile know if we're gcc # test_util.cc takes forever to compile with GCC and optimization turned on. diff --git a/java/pom.xml b/java/pom.xml index bbadc656..75d4c7f6 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -10,7 +10,7 @@ </parent> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> - <version>3.0.0-pre</version> + <version>3.0.0-alpha-2</version> <packaging>bundle</packaging> <name>Protocol Buffer Java API</name> <description> @@ -152,7 +152,7 @@ <instructions> <Bundle-DocURL>https://developers.google.com/protocol-buffers/</Bundle-DocURL> <Bundle-SymbolicName>com.google.protobuf</Bundle-SymbolicName> - <Export-Package>com.google.protobuf;version=3.0.0-pre</Export-Package> + <Export-Package>com.google.protobuf;version=3.0.0-alpha-2</Export-Package> </instructions> </configuration> </plugin> diff --git a/java/src/main/java/com/google/protobuf/GeneratedMessage.java b/java/src/main/java/com/google/protobuf/GeneratedMessage.java index 156d1633..d8510cb5 100644 --- a/java/src/main/java/com/google/protobuf/GeneratedMessage.java +++ b/java/src/main/java/com/google/protobuf/GeneratedMessage.java @@ -73,7 +73,7 @@ public abstract class GeneratedMessage extends AbstractMessage /** For use by generated code only. */ protected UnknownFieldSet unknownFields; - + protected GeneratedMessage() { unknownFields = UnknownFieldSet.getDefaultInstance(); } @@ -549,12 +549,12 @@ public abstract class GeneratedMessage extends AbstractMessage * Gets the map field with the given field number. This method should be * overridden in the generated message class if the message contains map * fields. - * + * * Unlike other field types, reflection support for map fields can't be * implemented based on generated public API because we need to access a * map field as a list in reflection API but the generated API only allows * us to access it as a map. This method returns the underlying map field - * directly and thus enables us to access the map field as a list. + * directly and thus enables us to access the map field as a list. */ @SuppressWarnings({"unused", "rawtypes"}) protected MapField internalGetMapField(int fieldNumber) { @@ -683,7 +683,7 @@ public abstract class GeneratedMessage extends AbstractMessage public final <Type> Type getExtension( final ExtensionLite<MessageType, Type> extensionLite) { Extension<MessageType, Type> extension = checkNotLite(extensionLite); - + verifyExtensionContainingType(extension); FieldDescriptor descriptor = extension.getDescriptor(); final Object value = extensions.getField(descriptor); @@ -1313,7 +1313,7 @@ public abstract class GeneratedMessage extends AbstractMessage implements ExtensionDescriptorRetriever { private volatile FieldDescriptor descriptor; protected abstract FieldDescriptor loadDescriptor(); - + public FieldDescriptor getDescriptor() { if (descriptor == null) { synchronized (this) { @@ -1651,17 +1651,17 @@ public abstract class GeneratedMessage extends AbstractMessage } } } - + /** * Gets the map field with the given field number. This method should be * overridden in the generated message class if the message contains map * fields. - * + * * Unlike other field types, reflection support for map fields can't be * implemented based on generated public API because we need to access a * map field as a list in reflection API but the generated API only allows * us to access it as a map. This method returns the underlying map field - * directly and thus enables us to access the map field as a list. + * directly and thus enables us to access the map field as a list. */ @SuppressWarnings({"rawtypes", "unused"}) protected MapField internalGetMapField(int fieldNumber) { @@ -1709,7 +1709,7 @@ public abstract class GeneratedMessage extends AbstractMessage oneofs = new OneofAccessor[descriptor.getOneofs().size()]; initialized = false; } - + private boolean isMapFieldEnabled(FieldDescriptor field) { boolean result = true; return result; @@ -1934,11 +1934,11 @@ public abstract class GeneratedMessage extends AbstractMessage protected final FieldDescriptor field; protected final boolean isOneofField; protected final boolean hasHasMethod; - + private int getOneofFieldNumber(final GeneratedMessage message) { return ((Internal.EnumLite) invokeOrDie(caseMethod, message)).getNumber(); } - + private int getOneofFieldNumber(final GeneratedMessage.Builder builder) { return ((Internal.EnumLite) invokeOrDie(caseMethodBuilder, builder)).getNumber(); } @@ -2130,15 +2130,15 @@ public abstract class GeneratedMessage extends AbstractMessage private final FieldDescriptor field; private final Message mapEntryMessageDefaultInstance; - + private MapField<?, ?> getMapField(GeneratedMessage message) { return (MapField<?, ?>) message.internalGetMapField(field.getNumber()); } - + private MapField<?, ?> getMapField(GeneratedMessage.Builder builder) { return (MapField<?, ?>) builder.internalGetMapField(field.getNumber()); } - + public Object get(GeneratedMessage message) { List result = new ArrayList(); for (int i = 0; i < getRepeatedCount(message); i++) { @@ -2171,10 +2171,12 @@ public abstract class GeneratedMessage extends AbstractMessage } public void setRepeated(Builder builder, int index, Object value) { + builder.onChanged(); getMapField(builder).getMutableList().set(index, (Message) value); } public void addRepeated(Builder builder, Object value) { + builder.onChanged(); getMapField(builder).getMutableList().add((Message) value); } @@ -2197,6 +2199,7 @@ public abstract class GeneratedMessage extends AbstractMessage } public void clear(Builder builder) { + builder.onChanged(); getMapField(builder).getMutableList().clear(); } @@ -2208,7 +2211,7 @@ public abstract class GeneratedMessage extends AbstractMessage throw new UnsupportedOperationException( "Nested builder not supported for map fields."); } - + public com.google.protobuf.Message.Builder getRepeatedBuilder( Builder builder, int index) { throw new UnsupportedOperationException( @@ -2226,7 +2229,7 @@ public abstract class GeneratedMessage extends AbstractMessage final Class<? extends Builder> builderClass, final String containingOneofCamelCaseName) { super(descriptor, camelCaseName, messageClass, builderClass, containingOneofCamelCaseName); - + enumDescriptor = descriptor.getEnumType(); valueOfMethod = getMethodOrDie(type, "valueOf", @@ -2244,12 +2247,12 @@ public abstract class GeneratedMessage extends AbstractMessage getMethodOrDie(builderClass, "set" + camelCaseName + "Value", int.class); } } - + private EnumDescriptor enumDescriptor; private Method valueOfMethod; private Method getValueDescriptorMethod; - + private boolean supportUnknownEnumValue; private Method getValueMethod; private Method getValueMethodBuilder; @@ -2291,7 +2294,7 @@ public abstract class GeneratedMessage extends AbstractMessage final Class<? extends GeneratedMessage> messageClass, final Class<? extends Builder> builderClass) { super(descriptor, camelCaseName, messageClass, builderClass); - + enumDescriptor = descriptor.getEnumType(); valueOfMethod = getMethodOrDie(type, "valueOf", @@ -2315,7 +2318,7 @@ public abstract class GeneratedMessage extends AbstractMessage private final Method valueOfMethod; private final Method getValueDescriptorMethod; - + private boolean supportUnknownEnumValue; private Method getRepeatedValueMethod; private Method getRepeatedValueMethodBuilder; @@ -2395,7 +2398,8 @@ public abstract class GeneratedMessage extends AbstractMessage final Class<? extends GeneratedMessage> messageClass, final Class<? extends Builder> builderClass, final String containingOneofCamelCaseName) { - super(descriptor, camelCaseName, messageClass, builderClass, containingOneofCamelCaseName); + super(descriptor, camelCaseName, messageClass, builderClass, + containingOneofCamelCaseName); newBuilderMethod = getMethodOrDie(type, "newBuilder"); getBuilderMethodBuilder = @@ -2492,7 +2496,7 @@ public abstract class GeneratedMessage extends AbstractMessage protected Object writeReplace() throws ObjectStreamException { return new GeneratedMessageLite.SerializedForm(this); } - + /** * Checks that the {@link Extension} is non-Lite and returns it as a * {@link GeneratedExtension}. @@ -2503,7 +2507,7 @@ public abstract class GeneratedMessage extends AbstractMessage if (extension.isLite()) { throw new IllegalArgumentException("Expected non-lite extension."); } - + return (Extension<MessageType, T>) extension; } } diff --git a/java/src/main/java/com/google/protobuf/Parser.java b/java/src/main/java/com/google/protobuf/Parser.java index f636014b..227c02b7 100644 --- a/java/src/main/java/com/google/protobuf/Parser.java +++ b/java/src/main/java/com/google/protobuf/Parser.java @@ -227,8 +227,8 @@ public interface Parser<MessageType> { * {@link MessageLite#writeDelimitedTo(java.io.OutputStream)} to write * messages in this format. * - * @return True if successful, or false if the stream is at EOF when the - * method starts. Any other error (including reaching EOF during + * @return Parsed message if successful, or null if the stream is at EOF when + * the method starts. Any other error (including reaching EOF during * parsing) will cause an exception to be thrown. */ public MessageType parseDelimitedFrom(InputStream input) diff --git a/java/src/test/java/com/google/protobuf/GeneratedMessageTest.java b/java/src/test/java/com/google/protobuf/GeneratedMessageTest.java index 41ed7bd0..2d101ba7 100644 --- a/java/src/test/java/com/google/protobuf/GeneratedMessageTest.java +++ b/java/src/test/java/com/google/protobuf/GeneratedMessageTest.java @@ -56,9 +56,10 @@ import protobuf_unittest.UnittestProto; import protobuf_unittest.UnittestProto.ForeignEnum; import protobuf_unittest.UnittestProto.ForeignMessage; import protobuf_unittest.UnittestProto.ForeignMessageOrBuilder; +import protobuf_unittest.UnittestProto.NestedTestAllTypes; import protobuf_unittest.UnittestProto.TestAllExtensions; -import protobuf_unittest.UnittestProto.TestAllTypes; import protobuf_unittest.UnittestProto.TestAllTypes.NestedMessage; +import protobuf_unittest.UnittestProto.TestAllTypes; import protobuf_unittest.UnittestProto.TestAllTypesOrBuilder; import protobuf_unittest.UnittestProto.TestExtremeDefaultValues; import protobuf_unittest.UnittestProto.TestOneof2; @@ -1510,6 +1511,17 @@ public class GeneratedMessageTest extends TestCase { } } + public void testOneofNestedBuilderOnChangePropagation() { + NestedTestAllTypes.Builder parentBuilder = NestedTestAllTypes.newBuilder(); + TestAllTypes.Builder builder = parentBuilder.getPayloadBuilder(); + builder.getOneofNestedMessageBuilder(); + assertTrue(builder.hasOneofNestedMessage()); + assertTrue(parentBuilder.hasPayload()); + NestedTestAllTypes message = parentBuilder.build(); + assertTrue(message.hasPayload()); + assertTrue(message.getPayload().hasOneofNestedMessage()); + } + public void testGetRepeatedFieldBuilder() { Descriptor descriptor = TestAllTypes.getDescriptor(); diff --git a/java/src/test/java/com/google/protobuf/MapTest.java b/java/src/test/java/com/google/protobuf/MapTest.java index 9a25e302..6a1e9078 100644 --- a/java/src/test/java/com/google/protobuf/MapTest.java +++ b/java/src/test/java/com/google/protobuf/MapTest.java @@ -57,22 +57,22 @@ public class MapTest extends TestCase { builder.getMutableInt32ToStringField().put(1, "11"); builder.getMutableInt32ToStringField().put(2, "22"); builder.getMutableInt32ToStringField().put(3, "33"); - + builder.getMutableInt32ToBytesField().put(1, TestUtil.toBytes("11")); builder.getMutableInt32ToBytesField().put(2, TestUtil.toBytes("22")); builder.getMutableInt32ToBytesField().put(3, TestUtil.toBytes("33")); - + builder.getMutableInt32ToEnumField().put(1, TestMap.EnumValue.FOO); builder.getMutableInt32ToEnumField().put(2, TestMap.EnumValue.BAR); builder.getMutableInt32ToEnumField().put(3, TestMap.EnumValue.BAZ); - + builder.getMutableInt32ToMessageField().put( 1, MessageValue.newBuilder().setValue(11).build()); builder.getMutableInt32ToMessageField().put( 2, MessageValue.newBuilder().setValue(22).build()); builder.getMutableInt32ToMessageField().put( 3, MessageValue.newBuilder().setValue(33).build()); - + builder.getMutableStringToInt32Field().put("1", 11); builder.getMutableStringToInt32Field().put("2", 22); builder.getMutableStringToInt32Field().put("3", 33); @@ -88,22 +88,22 @@ public class MapTest extends TestCase { assertEquals("11", message.getInt32ToStringField().get(1)); assertEquals("22", message.getInt32ToStringField().get(2)); assertEquals("33", message.getInt32ToStringField().get(3)); - + assertEquals(3, message.getInt32ToBytesField().size()); assertEquals(TestUtil.toBytes("11"), message.getInt32ToBytesField().get(1)); assertEquals(TestUtil.toBytes("22"), message.getInt32ToBytesField().get(2)); assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesField().get(3)); - + assertEquals(3, message.getInt32ToEnumField().size()); assertEquals(TestMap.EnumValue.FOO, message.getInt32ToEnumField().get(1)); assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(2)); assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumField().get(3)); - + assertEquals(3, message.getInt32ToMessageField().size()); assertEquals(11, message.getInt32ToMessageField().get(1).getValue()); assertEquals(22, message.getInt32ToMessageField().get(2).getValue()); assertEquals(33, message.getInt32ToMessageField().get(3).getValue()); - + assertEquals(3, message.getStringToInt32Field().size()); assertEquals(11, message.getStringToInt32Field().get("1").intValue()); assertEquals(22, message.getStringToInt32Field().get("2").intValue()); @@ -118,21 +118,21 @@ public class MapTest extends TestCase { builder.getMutableInt32ToStringField().put(1, "111"); builder.getMutableInt32ToStringField().remove(2); builder.getMutableInt32ToStringField().put(4, "44"); - + builder.getMutableInt32ToBytesField().put(1, TestUtil.toBytes("111")); builder.getMutableInt32ToBytesField().remove(2); builder.getMutableInt32ToBytesField().put(4, TestUtil.toBytes("44")); - + builder.getMutableInt32ToEnumField().put(1, TestMap.EnumValue.BAR); builder.getMutableInt32ToEnumField().remove(2); builder.getMutableInt32ToEnumField().put(4, TestMap.EnumValue.QUX); - + builder.getMutableInt32ToMessageField().put( 1, MessageValue.newBuilder().setValue(111).build()); builder.getMutableInt32ToMessageField().remove(2); builder.getMutableInt32ToMessageField().put( 4, MessageValue.newBuilder().setValue(44).build()); - + builder.getMutableStringToInt32Field().put("1", 111); builder.getMutableStringToInt32Field().remove("2"); builder.getMutableStringToInt32Field().put("4", 44); @@ -148,22 +148,22 @@ public class MapTest extends TestCase { assertEquals("111", message.getInt32ToStringField().get(1)); assertEquals("33", message.getInt32ToStringField().get(3)); assertEquals("44", message.getInt32ToStringField().get(4)); - + assertEquals(3, message.getInt32ToBytesField().size()); assertEquals(TestUtil.toBytes("111"), message.getInt32ToBytesField().get(1)); assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesField().get(3)); assertEquals(TestUtil.toBytes("44"), message.getInt32ToBytesField().get(4)); - + assertEquals(3, message.getInt32ToEnumField().size()); assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(1)); assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumField().get(3)); assertEquals(TestMap.EnumValue.QUX, message.getInt32ToEnumField().get(4)); - + assertEquals(3, message.getInt32ToMessageField().size()); assertEquals(111, message.getInt32ToMessageField().get(1).getValue()); assertEquals(33, message.getInt32ToMessageField().get(3).getValue()); assertEquals(44, message.getInt32ToMessageField().get(4).getValue()); - + assertEquals(3, message.getStringToInt32Field().size()); assertEquals(111, message.getStringToInt32Field().get("1").intValue()); assertEquals(33, message.getStringToInt32Field().get("3").intValue()); @@ -183,17 +183,17 @@ public class MapTest extends TestCase { TestMap.Builder builder = TestMap.newBuilder(); TestMap message = builder.build(); assertMapValuesCleared(message); - + builder = message.toBuilder(); setMapValues(builder); message = builder.build(); assertMapValuesSet(message); - + builder = message.toBuilder(); updateMapValues(builder); message = builder.build(); assertMapValuesUpdated(message); - + builder = message.toBuilder(); builder.clear(); message = builder.build(); @@ -207,14 +207,14 @@ public class MapTest extends TestCase { assertEquals(message.getSerializedSize(), message.toByteString().size()); message = TestMap.PARSER.parseFrom(message.toByteString()); assertMapValuesSet(message); - + builder = message.toBuilder(); updateMapValues(builder); message = builder.build(); assertEquals(message.getSerializedSize(), message.toByteString().size()); message = TestMap.PARSER.parseFrom(message.toByteString()); assertMapValuesUpdated(message); - + builder = message.toBuilder(); builder.clear(); message = builder.build(); @@ -222,12 +222,12 @@ public class MapTest extends TestCase { message = TestMap.PARSER.parseFrom(message.toByteString()); assertMapValuesCleared(message); } - + public void testMergeFrom() throws Exception { TestMap.Builder builder = TestMap.newBuilder(); setMapValues(builder); TestMap message = builder.build(); - + TestMap.Builder other = TestMap.newBuilder(); other.mergeFrom(message); assertMapValuesSet(other.build()); @@ -236,7 +236,7 @@ public class MapTest extends TestCase { public void testEqualsAndHashCode() throws Exception { // Test that generated equals() and hashCode() will disregard the order // of map entries when comparing/hashing map fields. - + // We can't control the order of elements in a HashMap. The best we can do // here is to add elements in different order. TestMap.Builder b1 = TestMap.newBuilder(); @@ -244,23 +244,23 @@ public class MapTest extends TestCase { b1.getMutableInt32ToInt32Field().put(3, 4); b1.getMutableInt32ToInt32Field().put(5, 6); TestMap m1 = b1.build(); - + TestMap.Builder b2 = TestMap.newBuilder(); b2.getMutableInt32ToInt32Field().put(5, 6); b2.getMutableInt32ToInt32Field().put(1, 2); b2.getMutableInt32ToInt32Field().put(3, 4); TestMap m2 = b2.build(); - + assertEquals(m1, m2); assertEquals(m1.hashCode(), m2.hashCode()); - + // Make sure we did compare map fields. b2.getMutableInt32ToInt32Field().put(1, 0); m2 = b2.build(); assertFalse(m1.equals(m2)); // Don't check m1.hashCode() != m2.hashCode() because it's not guaranteed // to be different. - + // Regression test for b/18549190: if a map is a subset of the other map, // equals() should return false. b2.getMutableInt32ToInt32Field().remove(1); @@ -268,57 +268,96 @@ public class MapTest extends TestCase { assertFalse(m1.equals(m2)); assertFalse(m2.equals(m1)); } - - + + public void testNestedBuilderOnChangeEventPropagation() { TestOnChangeEventPropagation.Builder parent = TestOnChangeEventPropagation.newBuilder(); parent.getOptionalMessageBuilder().getMutableInt32ToInt32Field().put(1, 2); TestOnChangeEventPropagation message = parent.build(); assertEquals(2, message.getOptionalMessage().getInt32ToInt32Field().get(1).intValue()); - + // Make a change using nested builder. parent.getOptionalMessageBuilder().getMutableInt32ToInt32Field().put(1, 3); - + // Should be able to observe the change. message = parent.build(); assertEquals(3, message.getOptionalMessage().getInt32ToInt32Field().get(1).intValue()); - + // Make another change using mergeFrom() TestMap.Builder other = TestMap.newBuilder(); other.getMutableInt32ToInt32Field().put(1, 4); parent.getOptionalMessageBuilder().mergeFrom(other.build()); - + // Should be able to observe the change. message = parent.build(); assertEquals(4, message.getOptionalMessage().getInt32ToInt32Field().get(1).intValue()); - + // Make yet another change by clearing the nested builder. parent.getOptionalMessageBuilder().clear(); - + // Should be able to observe the change. message = parent.build(); assertEquals(0, message.getOptionalMessage().getInt32ToInt32Field().size()); } - + + public void testNestedBuilderOnChangeEventPropagationReflection() { + FieldDescriptor intMapField = f("int32_to_int32_field"); + // Create an outer message builder with nested builder. + TestOnChangeEventPropagation.Builder parentBuilder = + TestOnChangeEventPropagation.newBuilder(); + TestMap.Builder testMapBuilder = parentBuilder.getOptionalMessageBuilder(); + + // Create a map entry message. + TestMap.Builder entryBuilder = TestMap.newBuilder(); + entryBuilder.getMutableInt32ToInt32Field().put(1, 1); + + // Put the entry into the nested builder. + testMapBuilder.addRepeatedField( + intMapField, entryBuilder.getRepeatedField(intMapField, 0)); + + // Should be able to observe the change. + TestOnChangeEventPropagation message = parentBuilder.build(); + assertEquals(1, message.getOptionalMessage().getInt32ToInt32Field().size()); + + // Change the entry value. + entryBuilder.getMutableInt32ToInt32Field().put(1, 4); + testMapBuilder = parentBuilder.getOptionalMessageBuilder(); + testMapBuilder.setRepeatedField( + intMapField, 0, entryBuilder.getRepeatedField(intMapField, 0)); + + // Should be able to observe the change. + message = parentBuilder.build(); + assertEquals(4, + message.getOptionalMessage().getInt32ToInt32Field().get(1).intValue()); + + // Clear the nested builder. + testMapBuilder = parentBuilder.getOptionalMessageBuilder(); + testMapBuilder.clearField(intMapField); + + // Should be able to observe the change. + message = parentBuilder.build(); + assertEquals(0, message.getOptionalMessage().getInt32ToInt32Field().size()); + } + // The following methods are used to test reflection API. - + private static FieldDescriptor f(String name) { return TestMap.getDescriptor().findFieldByName(name); } - + private static Object getFieldValue(Message mapEntry, String name) { FieldDescriptor field = mapEntry.getDescriptorForType().findFieldByName(name); return mapEntry.getField(field); } - + private static Message.Builder setFieldValue( Message.Builder mapEntry, String name, Object value) { FieldDescriptor field = mapEntry.getDescriptorForType().findFieldByName(name); mapEntry.setField(field, value); return mapEntry; } - + private static void assertHasMapValues(Message message, String name, Map<?, ?> values) { FieldDescriptor field = f(name); for (Object entry : (List<?>) message.getField(field)) { @@ -337,7 +376,7 @@ public class MapTest extends TestCase { assertEquals(value, values.get(key)); } } - + private static <KeyType, ValueType> Message newMapEntry(Message.Builder builder, String name, KeyType key, ValueType value) { FieldDescriptor field = builder.getDescriptorForType().findFieldByName(name); @@ -348,7 +387,7 @@ public class MapTest extends TestCase { entryBuilder.setField(valueField, value); return entryBuilder.build(); } - + private static void setMapValues(Message.Builder builder, String name, Map<?, ?> values) { List<Message> entryList = new ArrayList<Message>(); for (Map.Entry<?, ?> entry : values.entrySet()) { @@ -357,7 +396,7 @@ public class MapTest extends TestCase { FieldDescriptor field = builder.getDescriptorForType().findFieldByName(name); builder.setField(field, entryList); } - + private static <KeyType, ValueType> Map<KeyType, ValueType> mapForValues( KeyType key1, ValueType value1, KeyType key2, ValueType value2) { @@ -385,14 +424,14 @@ public class MapTest extends TestCase { mapForValues( 11, MessageValue.newBuilder().setValue(22).build(), 33, MessageValue.newBuilder().setValue(44).build())); - + // Test clearField() builder.clearField(f("int32_to_int32_field")); builder.clearField(f("int32_to_message_field")); message = builder.build(); assertEquals(0, message.getInt32ToInt32Field().size()); assertEquals(0, message.getInt32ToMessageField().size()); - + // Test setField() setMapValues(builder, "int32_to_int32_field", mapForValues(11, 22, 33, 44)); @@ -405,7 +444,7 @@ public class MapTest extends TestCase { assertEquals(44, message.getInt32ToInt32Field().get(33).intValue()); assertEquals(222, message.getInt32ToMessageField().get(111).getValue()); assertEquals(444, message.getInt32ToMessageField().get(333).getValue()); - + // Test addRepeatedField builder.addRepeatedField(f("int32_to_int32_field"), newMapEntry(builder, "int32_to_int32_field", 55, 66)); @@ -425,7 +464,7 @@ public class MapTest extends TestCase { message = builder.build(); assertEquals(55, message.getInt32ToInt32Field().get(55).intValue()); assertEquals(555, message.getInt32ToMessageField().get(555).getValue()); - + // Test setRepeatedField for (int i = 0; i < builder.getRepeatedFieldCount(f("int32_to_int32_field")); i++) { Message mapEntry = (Message) builder.getRepeatedField(f("int32_to_int32_field"), i); @@ -442,35 +481,35 @@ public class MapTest extends TestCase { assertEquals(33, message.getInt32ToInt32Field().get(44).intValue()); assertEquals(55, message.getInt32ToInt32Field().get(55).intValue()); } - + public void testTextFormat() throws Exception { TestMap.Builder builder = TestMap.newBuilder(); setMapValues(builder); TestMap message = builder.build(); - + String textData = TextFormat.printToString(message); - + builder = TestMap.newBuilder(); TextFormat.merge(textData, builder); message = builder.build(); - + assertMapValuesSet(message); } - + public void testDynamicMessage() throws Exception { TestMap.Builder builder = TestMap.newBuilder(); setMapValues(builder); TestMap message = builder.build(); - + Message dynamicDefaultInstance = DynamicMessage.getDefaultInstance(TestMap.getDescriptor()); Message dynamicMessage = dynamicDefaultInstance .newBuilderForType().mergeFrom(message.toByteString()).build(); - + assertEquals(message, dynamicMessage); assertEquals(message.hashCode(), dynamicMessage.hashCode()); } - + public void testReflectionEqualsAndHashCode() throws Exception { // Test that generated equals() and hashCode() will disregard the order // of map entries when comparing/hashing map fields. @@ -479,22 +518,22 @@ public class MapTest extends TestCase { Message dynamicDefaultInstance = DynamicMessage.getDefaultInstance(TestMap.getDescriptor()); FieldDescriptor field = f("int32_to_int32_field"); - + Message.Builder b1 = dynamicDefaultInstance.newBuilderForType(); b1.addRepeatedField(field, newMapEntry(b1, "int32_to_int32_field", 1, 2)); b1.addRepeatedField(field, newMapEntry(b1, "int32_to_int32_field", 3, 4)); b1.addRepeatedField(field, newMapEntry(b1, "int32_to_int32_field", 5, 6)); Message m1 = b1.build(); - + Message.Builder b2 = dynamicDefaultInstance.newBuilderForType(); b2.addRepeatedField(field, newMapEntry(b2, "int32_to_int32_field", 5, 6)); b2.addRepeatedField(field, newMapEntry(b2, "int32_to_int32_field", 1, 2)); b2.addRepeatedField(field, newMapEntry(b2, "int32_to_int32_field", 3, 4)); Message m2 = b2.build(); - + assertEquals(m1, m2); assertEquals(m1.hashCode(), m2.hashCode()); - + // Make sure we did compare map fields. b2.setRepeatedField(field, 0, newMapEntry(b1, "int32_to_int32_field", 0, 0)); m2 = b2.build(); @@ -502,7 +541,7 @@ public class MapTest extends TestCase { // Don't check m1.hashCode() != m2.hashCode() because it's not guaranteed // to be different. } - + public void testUnknownEnumValues() throws Exception { TestMap.Builder builder = TestMap.newBuilder(); builder.getMutableInt32ToEnumFieldValue().put(0, 0); @@ -517,7 +556,7 @@ public class MapTest extends TestCase { assertEquals(TestMap.EnumValue.UNRECOGNIZED, message.getInt32ToEnumField().get(2)); assertEquals(1000, message.getInt32ToEnumFieldValue().get(2).intValue()); - + // Unknown enum values should be preserved after: // 1. Serialization and parsing. // 2. toBuild(). @@ -528,7 +567,7 @@ public class MapTest extends TestCase { assertEquals(1000, builder.getInt32ToEnumFieldValue().get(2).intValue()); builder = TestMap.newBuilder().mergeFrom(message); assertEquals(1000, builder.getInt32ToEnumFieldValue().get(2).intValue()); - + // hashCode()/equals() should take unknown enum values into account. builder.getMutableInt32ToEnumFieldValue().put(2, 1001); TestMap message2 = builder.build(); @@ -538,17 +577,17 @@ public class MapTest extends TestCase { // should be the same. assertTrue(message.getInt32ToEnumField().equals(message2.getInt32ToEnumField())); } - + public void testUnknownEnumValuesInReflectionApi() throws Exception { Descriptor descriptor = TestMap.getDescriptor(); EnumDescriptor enumDescriptor = TestMap.EnumValue.getDescriptor(); FieldDescriptor field = descriptor.findFieldByName("int32_to_enum_field"); - + Map<Integer, Integer> data = new HashMap<Integer, Integer>(); data.put(0, 0); data.put(1, 1); data.put(2, 1000); // unknown value. - + TestMap.Builder builder = TestMap.newBuilder(); for (Map.Entry<Integer, Integer> entry : data.entrySet()) { builder.getMutableInt32ToEnumFieldValue().put(entry.getKey(), entry.getValue()); diff --git a/javanano/README.txt b/javanano/README.txt index 5a05b865..cfb3c3b4 100644 --- a/javanano/README.txt +++ b/javanano/README.txt @@ -68,18 +68,20 @@ running unit tests. Nano version ============================ -Nano is a special code generator and runtime library designed specially -for Android, and is very resource-friendly in both the amount of code -and the runtime overhead. An overview of Nano features: +JavaNano is a special code generator and runtime library designed specially for +resource-restricted systems, like Android. It is very resource-friendly in both +the amount of code and the runtime overhead. Here is an overview of JavaNano +features compared with the official Java protobuf: - No descriptors or message builders. - All messages are mutable; fields are public Java fields. - For optional fields only, encapsulation behind setter/getter/hazzer/ clearer functions is opt-in, which provide proper 'has' state support. -- If not opted in, has state is not available. Serialization outputs - all fields not equal to their defaults (see important implications - below). -- Required fields are always serialized. +- For proto2, if not opted in, has state (field presence) is not available. + Serialization outputs all fields not equal to their defaults + (see important implications below). + The behavior is consistent with proto3 semantics. +- Required fields (proto2 only) are always serialized. - Enum constants are integers; protection against invalid values only when parsing from the wire. - Enum constants can be generated into container interfaces bearing @@ -88,8 +90,8 @@ and the runtime overhead. An overview of Nano features: - Similarly CodedOutputByteBufferNano can only write to byte[]. - Repeated fields are in arrays, not ArrayList or Vector. Null array elements are allowed and silently ignored. -- Full support of serializing/deserializing repeated packed fields. -- Support of extensions. +- Full support for serializing/deserializing repeated packed fields. +- Support extensions (in proto2). - Unset messages/groups are null, not an immutable empty default instance. - toByteArray(...) and mergeFrom(...) are now static functions of @@ -200,7 +202,7 @@ optional_field_style={default,accessors,reftypes} (default: default) In the default style, optional fields translate into public mutable Java fields, and the serialization process is as discussed in the - "IMPORTANT" section above. + "IMPORTANT" section above. * accessors * diff --git a/javanano/pom.xml b/javanano/pom.xml index 3d98a5e0..50056cd3 100644 --- a/javanano/pom.xml +++ b/javanano/pom.xml @@ -10,7 +10,7 @@ </parent> <groupId>com.google.protobuf.nano</groupId> <artifactId>protobuf-javanano</artifactId> - <version>2.6.2-pre</version> + <version>3.0.0-alpha-2</version> <packaging>bundle</packaging> <name>Protocol Buffer JavaNano API</name> <description> @@ -94,6 +94,7 @@ <arg value="src/test/java/com/google/protobuf/nano/unittest_multiple_nameclash_nano.proto" /> <arg value="src/test/java/com/google/protobuf/nano/unittest_enum_class_nano.proto" /> <arg value="src/test/java/com/google/protobuf/nano/unittest_repeated_merge_nano.proto" /> + <arg value="src/test/java/com/google/protobuf/nano/map_test.proto" /> </exec> <exec executable="../src/protoc"> <arg value="--javanano_out=store_unknown_fields=true,generate_equals=true:target/generated-test-sources" /> @@ -155,7 +156,7 @@ <instructions> <Bundle-DocURL>https://developers.google.com/protocol-buffers/</Bundle-DocURL> <Bundle-SymbolicName>com.google.protobuf</Bundle-SymbolicName> - <Export-Package>com.google.protobuf;version=2.6.2-pre</Export-Package> + <Export-Package>com.google.protobuf;version=3.0.0-alpha-2</Export-Package> </instructions> </configuration> </plugin> diff --git a/javanano/src/main/java/com/google/protobuf/nano/CodedInputByteBufferNano.java b/javanano/src/main/java/com/google/protobuf/nano/CodedInputByteBufferNano.java index b147f69d..b4f20fde 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/CodedInputByteBufferNano.java +++ b/javanano/src/main/java/com/google/protobuf/nano/CodedInputByteBufferNano.java @@ -638,4 +638,44 @@ public final class CodedInputByteBufferNano { throw InvalidProtocolBufferNanoException.truncatedMessage(); } } + + // Read a primitive type. + Object readPrimitiveField(int type) throws IOException { + switch (type) { + case InternalNano.TYPE_DOUBLE: + return readDouble(); + case InternalNano.TYPE_FLOAT: + return readFloat(); + case InternalNano.TYPE_INT64: + return readInt64(); + case InternalNano.TYPE_UINT64: + return readUInt64(); + case InternalNano.TYPE_INT32: + return readInt32(); + case InternalNano.TYPE_FIXED64: + return readFixed64(); + case InternalNano.TYPE_FIXED32: + return readFixed32(); + case InternalNano.TYPE_BOOL: + return readBool(); + case InternalNano.TYPE_STRING: + return readString(); + case InternalNano.TYPE_BYTES: + return readBytes(); + case InternalNano.TYPE_UINT32: + return readUInt32(); + case InternalNano.TYPE_ENUM: + return readEnum(); + case InternalNano.TYPE_SFIXED32: + return readSFixed32(); + case InternalNano.TYPE_SFIXED64: + return readSFixed64(); + case InternalNano.TYPE_SINT32: + return readSInt32(); + case InternalNano.TYPE_SINT64: + return readSInt64(); + default: + throw new IllegalArgumentException("Unknown type " + type); + } + } } diff --git a/javanano/src/main/java/com/google/protobuf/nano/CodedOutputByteBufferNano.java b/javanano/src/main/java/com/google/protobuf/nano/CodedOutputByteBufferNano.java index 37982b57..2777f34c 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/CodedOutputByteBufferNano.java +++ b/javanano/src/main/java/com/google/protobuf/nano/CodedOutputByteBufferNano.java @@ -876,4 +876,128 @@ public final class CodedOutputByteBufferNano { // Note: the right-shift must be arithmetic return (n << 1) ^ (n >> 63); } + + static int computeFieldSize(int number, int type, Object object) { + switch (type) { + case InternalNano.TYPE_BOOL: + return computeBoolSize(number, (Boolean) object); + case InternalNano.TYPE_BYTES: + return computeBytesSize(number, (byte[]) object); + case InternalNano.TYPE_STRING: + return computeStringSize(number, (String) object); + case InternalNano.TYPE_FLOAT: + return computeFloatSize(number, (Float) object); + case InternalNano.TYPE_DOUBLE: + return computeDoubleSize(number, (Double) object); + case InternalNano.TYPE_ENUM: + return computeEnumSize(number, (Integer) object); + case InternalNano.TYPE_FIXED32: + return computeFixed32Size(number, (Integer) object); + case InternalNano.TYPE_INT32: + return computeInt32Size(number, (Integer) object); + case InternalNano.TYPE_UINT32: + return computeUInt32Size(number, (Integer) object); + case InternalNano.TYPE_SINT32: + return computeSInt32Size(number, (Integer) object); + case InternalNano.TYPE_SFIXED32: + return computeSFixed32Size(number, (Integer) object); + case InternalNano.TYPE_INT64: + return computeInt64Size(number, (Long) object); + case InternalNano.TYPE_UINT64: + return computeUInt64Size(number, (Long) object); + case InternalNano.TYPE_SINT64: + return computeSInt64Size(number, (Long) object); + case InternalNano.TYPE_FIXED64: + return computeFixed64Size(number, (Long) object); + case InternalNano.TYPE_SFIXED64: + return computeSFixed64Size(number, (Long) object); + case InternalNano.TYPE_MESSAGE: + return computeMessageSize(number, (MessageNano) object); + case InternalNano.TYPE_GROUP: + return computeGroupSize(number, (MessageNano) object); + default: + throw new IllegalArgumentException("Unknown type: " + type); + } + } + + void writeField(int number, int type, Object value) + throws IOException { + switch (type) { + case InternalNano.TYPE_DOUBLE: + Double doubleValue = (Double) value; + writeDouble(number, doubleValue); + break; + case InternalNano.TYPE_FLOAT: + Float floatValue = (Float) value; + writeFloat(number, floatValue); + break; + case InternalNano.TYPE_INT64: + Long int64Value = (Long) value; + writeInt64(number, int64Value); + break; + case InternalNano.TYPE_UINT64: + Long uint64Value = (Long) value; + writeUInt64(number, uint64Value); + break; + case InternalNano.TYPE_INT32: + Integer int32Value = (Integer) value; + writeInt32(number, int32Value); + break; + case InternalNano.TYPE_FIXED64: + Long fixed64Value = (Long) value; + writeFixed64(number, fixed64Value); + break; + case InternalNano.TYPE_FIXED32: + Integer fixed32Value = (Integer) value; + writeFixed32(number, fixed32Value); + break; + case InternalNano.TYPE_BOOL: + Boolean boolValue = (Boolean) value; + writeBool(number, boolValue); + break; + case InternalNano.TYPE_STRING: + String stringValue = (String) value; + writeString(number, stringValue); + break; + case InternalNano.TYPE_BYTES: + byte[] bytesValue = (byte[]) value; + writeBytes(number, bytesValue); + break; + case InternalNano.TYPE_UINT32: + Integer uint32Value = (Integer) value; + writeUInt32(number, uint32Value); + break; + case InternalNano.TYPE_ENUM: + Integer enumValue = (Integer) value; + writeEnum(number, enumValue); + break; + case InternalNano.TYPE_SFIXED32: + Integer sfixed32Value = (Integer) value; + writeSFixed32(number, sfixed32Value); + break; + case InternalNano.TYPE_SFIXED64: + Long sfixed64Value = (Long) value; + writeSFixed64(number, sfixed64Value); + break; + case InternalNano.TYPE_SINT32: + Integer sint32Value = (Integer) value; + writeSInt32(number, sint32Value); + break; + case InternalNano.TYPE_SINT64: + Long sint64Value = (Long) value; + writeSInt64(number, sint64Value); + break; + case InternalNano.TYPE_MESSAGE: + MessageNano messageValue = (MessageNano) value; + writeMessage(number, messageValue); + break; + case InternalNano.TYPE_GROUP: + MessageNano groupValue = (MessageNano) value; + writeGroup(number, groupValue); + break; + default: + throw new IOException("Unknown type: " + type); + } + } + } diff --git a/javanano/src/main/java/com/google/protobuf/nano/Extension.java b/javanano/src/main/java/com/google/protobuf/nano/Extension.java index 5d18ae6e..c29b030f 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/Extension.java +++ b/javanano/src/main/java/com/google/protobuf/nano/Extension.java @@ -55,24 +55,24 @@ public class Extension<M extends ExtendableMessageNano<M>, T> { * PrimitiveExtension // for primitive/enum typed extensions */ - public static final int TYPE_DOUBLE = 1; - public static final int TYPE_FLOAT = 2; - public static final int TYPE_INT64 = 3; - public static final int TYPE_UINT64 = 4; - public static final int TYPE_INT32 = 5; - public static final int TYPE_FIXED64 = 6; - public static final int TYPE_FIXED32 = 7; - public static final int TYPE_BOOL = 8; - public static final int TYPE_STRING = 9; - public static final int TYPE_GROUP = 10; - public static final int TYPE_MESSAGE = 11; - public static final int TYPE_BYTES = 12; - public static final int TYPE_UINT32 = 13; - public static final int TYPE_ENUM = 14; - public static final int TYPE_SFIXED32 = 15; - public static final int TYPE_SFIXED64 = 16; - public static final int TYPE_SINT32 = 17; - public static final int TYPE_SINT64 = 18; + public static final int TYPE_DOUBLE = InternalNano.TYPE_DOUBLE; + public static final int TYPE_FLOAT = InternalNano.TYPE_FLOAT; + public static final int TYPE_INT64 = InternalNano.TYPE_INT64; + public static final int TYPE_UINT64 = InternalNano.TYPE_UINT64; + public static final int TYPE_INT32 = InternalNano.TYPE_INT32; + public static final int TYPE_FIXED64 = InternalNano.TYPE_FIXED64; + public static final int TYPE_FIXED32 = InternalNano.TYPE_FIXED32; + public static final int TYPE_BOOL = InternalNano.TYPE_BOOL; + public static final int TYPE_STRING = InternalNano.TYPE_STRING; + public static final int TYPE_GROUP = InternalNano.TYPE_GROUP; + public static final int TYPE_MESSAGE = InternalNano.TYPE_MESSAGE; + public static final int TYPE_BYTES = InternalNano.TYPE_BYTES; + public static final int TYPE_UINT32 = InternalNano.TYPE_UINT32; + public static final int TYPE_ENUM = InternalNano.TYPE_ENUM; + public static final int TYPE_SFIXED32 = InternalNano.TYPE_SFIXED32; + public static final int TYPE_SFIXED64 = InternalNano.TYPE_SFIXED64; + public static final int TYPE_SINT32 = InternalNano.TYPE_SINT32; + public static final int TYPE_SINT64 = InternalNano.TYPE_SINT64; /** * Creates an {@code Extension} of the given message type and tag number. @@ -338,42 +338,7 @@ public class Extension<M extends ExtendableMessageNano<M>, T> { @Override protected Object readData(CodedInputByteBufferNano input) { try { - switch (type) { - case TYPE_DOUBLE: - return input.readDouble(); - case TYPE_FLOAT: - return input.readFloat(); - case TYPE_INT64: - return input.readInt64(); - case TYPE_UINT64: - return input.readUInt64(); - case TYPE_INT32: - return input.readInt32(); - case TYPE_FIXED64: - return input.readFixed64(); - case TYPE_FIXED32: - return input.readFixed32(); - case TYPE_BOOL: - return input.readBool(); - case TYPE_STRING: - return input.readString(); - case TYPE_BYTES: - return input.readBytes(); - case TYPE_UINT32: - return input.readUInt32(); - case TYPE_ENUM: - return input.readEnum(); - case TYPE_SFIXED32: - return input.readSFixed32(); - case TYPE_SFIXED64: - return input.readSFixed64(); - case TYPE_SINT32: - return input.readSInt32(); - case TYPE_SINT64: - return input.readSInt64(); - default: - throw new IllegalArgumentException("Unknown type " + type); - } + return input.readPrimitiveField(type); } catch (IOException e) { throw new IllegalArgumentException("Error reading extension field", e); } diff --git a/javanano/src/main/java/com/google/protobuf/nano/InternalNano.java b/javanano/src/main/java/com/google/protobuf/nano/InternalNano.java index e08bb4b7..4a08bb58 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/InternalNano.java +++ b/javanano/src/main/java/com/google/protobuf/nano/InternalNano.java @@ -30,8 +30,13 @@ package com.google.protobuf.nano; +import com.google.protobuf.nano.MapFactories.MapFactory; + +import java.io.IOException; import java.io.UnsupportedEncodingException; import java.util.Arrays; +import java.util.Map; +import java.util.Map.Entry; /** * The classes contained within are used internally by the Protocol Buffer @@ -43,6 +48,26 @@ import java.util.Arrays; */ public final class InternalNano { + public static final int TYPE_DOUBLE = 1; + public static final int TYPE_FLOAT = 2; + public static final int TYPE_INT64 = 3; + public static final int TYPE_UINT64 = 4; + public static final int TYPE_INT32 = 5; + public static final int TYPE_FIXED64 = 6; + public static final int TYPE_FIXED32 = 7; + public static final int TYPE_BOOL = 8; + public static final int TYPE_STRING = 9; + public static final int TYPE_GROUP = 10; + public static final int TYPE_MESSAGE = 11; + public static final int TYPE_BYTES = 12; + public static final int TYPE_UINT32 = 13; + public static final int TYPE_ENUM = 14; + public static final int TYPE_SFIXED32 = 15; + public static final int TYPE_SFIXED64 = 16; + public static final int TYPE_SINT32 = 17; + public static final int TYPE_SINT64 = 18; + + private InternalNano() {} /** @@ -329,5 +354,202 @@ public final class InternalNano { } return result; } + private static Object primitiveDefaultValue(int type) { + switch (type) { + case TYPE_BOOL: + return Boolean.FALSE; + case TYPE_BYTES: + return WireFormatNano.EMPTY_BYTES; + case TYPE_STRING: + return ""; + case TYPE_FLOAT: + return Float.valueOf(0); + case TYPE_DOUBLE: + return Double.valueOf(0); + case TYPE_ENUM: + case TYPE_FIXED32: + case TYPE_INT32: + case TYPE_UINT32: + case TYPE_SINT32: + case TYPE_SFIXED32: + return Integer.valueOf(0); + case TYPE_INT64: + case TYPE_UINT64: + case TYPE_SINT64: + case TYPE_FIXED64: + case TYPE_SFIXED64: + return Long.valueOf(0L); + case TYPE_MESSAGE: + case TYPE_GROUP: + default: + throw new IllegalArgumentException( + "Type: " + type + " is not a primitive type."); + } + } + + /** + * Merges the map entry into the map field. Note this is only supposed to + * be called by generated messages. + * + * @param map the map field; may be null, in which case a map will be + * instantiated using the {@link MapFactories.MapFactory} + * @param input the input byte buffer + * @param keyType key type, as defined in InternalNano.TYPE_* + * @param valueType value type, as defined in InternalNano.TYPE_* + * @param value an new instance of the value, if the value is a TYPE_MESSAGE; + * otherwise this parameter can be null and will be ignored. + * @param keyTag wire tag for the key + * @param valueTag wire tag for the value + * @return the map field + * @throws IOException + */ + @SuppressWarnings("unchecked") + public static final <K, V> Map<K, V> mergeMapEntry( + CodedInputByteBufferNano input, + Map<K, V> map, + MapFactory mapFactory, + int keyType, + int valueType, + V value, + int keyTag, + int valueTag) throws IOException { + map = mapFactory.forMap(map); + final int length = input.readRawVarint32(); + final int oldLimit = input.pushLimit(length); + K key = null; + while (true) { + int tag = input.readTag(); + if (tag == 0) { + break; + } + if (tag == keyTag) { + key = (K) input.readPrimitiveField(keyType); + } else if (tag == valueTag) { + if (valueType == TYPE_MESSAGE) { + input.readMessage((MessageNano) value); + } else { + value = (V) input.readPrimitiveField(valueType); + } + } else { + if (!input.skipField(tag)) { + break; + } + } + } + input.checkLastTagWas(0); + input.popLimit(oldLimit); + + if (key == null) { + // key can only be primitive types. + key = (K) primitiveDefaultValue(keyType); + } + + if (value == null) { + // message type value will be initialized by code-gen. + value = (V) primitiveDefaultValue(valueType); + } + + map.put(key, value); + return map; + } + + public static <K, V> void serializeMapField( + CodedOutputByteBufferNano output, + Map<K, V> map, int number, int keyType, int valueType) + throws IOException { + for (Entry<K, V> entry: map.entrySet()) { + K key = entry.getKey(); + V value = entry.getValue(); + if (key == null || value == null) { + throw new IllegalStateException( + "keys and values in maps cannot be null"); + } + int entrySize = + CodedOutputByteBufferNano.computeFieldSize(1, keyType, key) + + CodedOutputByteBufferNano.computeFieldSize(2, valueType, value); + output.writeTag(number, WireFormatNano.WIRETYPE_LENGTH_DELIMITED); + output.writeRawVarint32(entrySize); + output.writeField(1, keyType, key); + output.writeField(2, valueType, value); + } + } + + public static <K, V> int computeMapFieldSize( + Map<K, V> map, int number, int keyType, int valueType) { + int size = 0; + int tagSize = CodedOutputByteBufferNano.computeTagSize(number); + for (Entry<K, V> entry: map.entrySet()) { + K key = entry.getKey(); + V value = entry.getValue(); + if (key == null || value == null) { + throw new IllegalStateException( + "keys and values in maps cannot be null"); + } + int entrySize = + CodedOutputByteBufferNano.computeFieldSize(1, keyType, key) + + CodedOutputByteBufferNano.computeFieldSize(2, valueType, value); + size += tagSize + entrySize + + CodedOutputByteBufferNano.computeRawVarint32Size(entrySize); + } + return size; + } + + /** + * Checks whether two {@link Map} are equal. We don't use the default equals + * method of {@link Map} because it compares by identity not by content for + * byte arrays. + */ + public static <K, V> boolean equals(Map<K, V> a, Map<K, V> b) { + if (a == b) { + return true; + } + if (a == null) { + return b.size() == 0; + } + if (b == null) { + return a.size() == 0; + } + if (a.size() != b.size()) { + return false; + } + for (Entry<K, V> entry : a.entrySet()) { + if (!b.containsKey(entry.getKey())) { + return false; + } + if (!equalsMapValue(entry.getValue(), b.get(entry.getKey()))) { + return false; + } + } + return true; + } + + private static boolean equalsMapValue(Object a, Object b) { + if (a == null || b == null) { + throw new IllegalStateException( + "keys and values in maps cannot be null"); + } + if (a instanceof byte[] && b instanceof byte[]) { + return Arrays.equals((byte[]) a, (byte[]) b); + } + return a.equals(b); + } + + public static <K, V> int hashCode(Map<K, V> map) { + if (map == null) { + return 0; + } + int result = 0; + for (Entry<K, V> entry : map.entrySet()) { + result += hashCodeForMap(entry.getKey()) + ^ hashCodeForMap(entry.getValue()); + } + return result; + } + private static int hashCodeForMap(Object o) { + if (o instanceof byte[]) { + return Arrays.hashCode((byte[]) o); + } + return o.hashCode(); + } } diff --git a/javanano/src/main/java/com/google/protobuf/nano/MapFactories.java b/javanano/src/main/java/com/google/protobuf/nano/MapFactories.java new file mode 100644 index 00000000..98fa4877 --- /dev/null +++ b/javanano/src/main/java/com/google/protobuf/nano/MapFactories.java @@ -0,0 +1,67 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2013 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf.nano; + +import java.util.HashMap; +import java.util.Map; + +/** + * Utility class for maps support. + */ +public final class MapFactories { + public static interface MapFactory { + <K, V> Map<K, V> forMap(Map<K, V> oldMap); + } + + // NOTE(liujisi): The factory setter is temporarily marked as package private. + // The way to provide customized implementations of maps for different + // platforms are still under discussion. Mark it as private to avoid exposing + // the API in proto3 alpha release. + /* public */ static void setMapFactory(MapFactory newMapFactory) { + mapFactory = newMapFactory; + } + + public static MapFactory getMapFactory() { + return mapFactory; + } + + private static class DefaultMapFactory implements MapFactory { + public <K, V> Map<K, V> forMap(Map<K, V> oldMap) { + if (oldMap == null) { + return new HashMap<K, V>(); + } + return oldMap; + } + } + private static volatile MapFactory mapFactory = new DefaultMapFactory(); + + private MapFactories() {} +} diff --git a/javanano/src/main/java/com/google/protobuf/nano/MessageNanoPrinter.java b/javanano/src/main/java/com/google/protobuf/nano/MessageNanoPrinter.java index dd43cdbb..c4b2ad3d 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/MessageNanoPrinter.java +++ b/javanano/src/main/java/com/google/protobuf/nano/MessageNanoPrinter.java @@ -35,6 +35,7 @@ import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; +import java.util.Map; /** * Static helper methods for printing nano protos. @@ -170,6 +171,19 @@ public final class MessageNanoPrinter { indentBuf.setLength(origIndentBufLength); buf.append(indentBuf).append(">\n"); } + } else if (object instanceof Map) { + Map<?,?> map = (Map<?,?>) object; + identifier = deCamelCaseify(identifier); + + for (Map.Entry<?,?> entry : map.entrySet()) { + buf.append(indentBuf).append(identifier).append(" <\n"); + int origIndentBufLength = indentBuf.length(); + indentBuf.append(INDENT); + print("key", entry.getKey(), indentBuf, buf); + print("value", entry.getValue(), indentBuf, buf); + indentBuf.setLength(origIndentBufLength); + buf.append(indentBuf).append(">\n"); + } } else { // Non-null primitive value identifier = deCamelCaseify(identifier); diff --git a/javanano/src/test/java/com/google/protobuf/nano/NanoTest.java b/javanano/src/test/java/com/google/protobuf/nano/NanoTest.java index 442f0b74..4cc77226 100644 --- a/javanano/src/test/java/com/google/protobuf/nano/NanoTest.java +++ b/javanano/src/test/java/com/google/protobuf/nano/NanoTest.java @@ -30,31 +30,11 @@ package com.google.protobuf.nano; -import com.google.protobuf.nano.CodedInputByteBufferNano; -import com.google.protobuf.nano.EnumClassNanoMultiple; -import com.google.protobuf.nano.EnumClassNanos; -import com.google.protobuf.nano.EnumValidity; -import com.google.protobuf.nano.EnumValidityAccessors; -import com.google.protobuf.nano.FileScopeEnumMultiple; -import com.google.protobuf.nano.FileScopeEnumRefNano; -import com.google.protobuf.nano.InternalNano; -import com.google.protobuf.nano.InvalidProtocolBufferNanoException; -import com.google.protobuf.nano.MessageNano; -import com.google.protobuf.nano.MessageScopeEnumRefNano; -import com.google.protobuf.nano.MultipleImportingNonMultipleNano1; -import com.google.protobuf.nano.MultipleImportingNonMultipleNano2; -import com.google.protobuf.nano.MultipleNameClashNano; +import com.google.protobuf.nano.MapTestProto.TestMap; +import com.google.protobuf.nano.MapTestProto.TestMap.MessageValue; import com.google.protobuf.nano.NanoAccessorsOuterClass.TestNanoAccessors; import com.google.protobuf.nano.NanoHasOuterClass.TestAllTypesNanoHas; -import com.google.protobuf.nano.NanoOuterClass; import com.google.protobuf.nano.NanoOuterClass.TestAllTypesNano; -import com.google.protobuf.nano.NanoReferenceTypes; -import com.google.protobuf.nano.NanoRepeatedPackables; -import com.google.protobuf.nano.PackedExtensions; -import com.google.protobuf.nano.RepeatedExtensions; -import com.google.protobuf.nano.SingularExtensions; -import com.google.protobuf.nano.TestRepeatedMergeNano; -import com.google.protobuf.nano.UnittestMultipleNano; import com.google.protobuf.nano.UnittestRecursiveNano.RecursiveMessageNano; import com.google.protobuf.nano.UnittestSimpleNano.SimpleMessageNano; import com.google.protobuf.nano.UnittestSingleNano.SingleMessageNano; @@ -67,6 +47,8 @@ import junit.framework.TestCase; import java.util.Arrays; import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; /** * Test nano runtime. @@ -2688,6 +2670,7 @@ public class NanoTest extends TestCase { msg.repeatedNestedEnum[0] = TestAllTypesNano.BAR; msg.repeatedNestedEnum[1] = TestAllTypesNano.FOO; msg.repeatedStringPiece = new String[] {null, "world"}; + msg.setOneofString("hello"); String protoPrint = msg.toString(); assertTrue(protoPrint.contains("optional_int32: 14")); @@ -2711,6 +2694,7 @@ public class NanoTest extends TestCase { assertTrue(protoPrint.contains("default_string: \"hello\"")); assertFalse(protoPrint.contains("repeated_string_piece: \"\"")); // null should be dropped assertTrue(protoPrint.contains("repeated_string_piece: \"world\"")); + assertTrue(protoPrint.contains("oneof_string: \"hello\"")); } public void testMessageNanoPrinterAccessors() throws Exception { @@ -2749,6 +2733,45 @@ public class NanoTest extends TestCase { assertTrue(protoPrint.contains("id: 33")); } + public void testMessageNanoPrinterForMaps() throws Exception { + TestMap msg = new TestMap(); + MessageValue msgValues[] = new MessageValue[] { + new MessageValue(), new MessageValue() + }; + msgValues[0].value = 1; + msgValues[1].value = 2; + msg.int32ToBytesField = new HashMap<Integer, byte[]>(); + msg.int32ToBytesField.put(1, new byte[] {'"', '\0'}); + msg.int32ToBytesField.put(2, new byte[] {1, 8}); + msg.stringToInt32Field = new HashMap<String, Integer>(); + msg.stringToInt32Field.put("hello", 1); + msg.stringToInt32Field.put("world", 2); + msg.int32ToMessageField = new HashMap<Integer, MapTestProto.TestMap.MessageValue>(); + msg.int32ToMessageField.put(0, msgValues[0]); + msg.int32ToMessageField.put(1, msgValues[1]); + msg.int32ToEnumField = new HashMap<Integer, Integer>(); + msg.int32ToEnumField.put(1, 2); + msg.int32ToEnumField.put(2, 3); + String protoPrint = msg.toString(); + + assertTrue(protoPrint.contains( + "int32_to_bytes_field <\n key: 1\n value: \"\\\"\\000\"\n>")); + assertTrue(protoPrint.contains( + "int32_to_bytes_field <\n key: 2\n value: \"\\001\\010\"\n>")); + assertTrue(protoPrint.contains( + "string_to_int32_field <\n key: \"hello\"\n value: 1\n>")); + assertTrue(protoPrint.contains( + "string_to_int32_field <\n key: \"world\"\n value: 2\n>")); + assertTrue(protoPrint.contains( + "int32_to_message_field <\n key: 0\n value <\n value: 1\n")); + assertTrue(protoPrint.contains( + "int32_to_message_field <\n key: 1\n value <\n value: 2\n")); + assertTrue(protoPrint.contains( + "int32_to_enum_field <\n key: 1\n value: 2\n>")); + assertTrue(protoPrint.contains( + "int32_to_enum_field <\n key: 2\n value: 3\n>")); + } + public void testExtensions() throws Exception { Extensions.ExtendableMessage message = new Extensions.ExtendableMessage(); message.field = 5; @@ -3340,6 +3363,7 @@ public class NanoTest extends TestCase { TestAllTypesNano.BAR, TestAllTypesNano.BAZ }; + message.setOneofUint32(3); return message; } @@ -3538,6 +3562,259 @@ public class NanoTest extends TestCase { new NanoReferenceTypes.TestAllTypesNano(), MessageNano.toByteArray(m7)))); } + private static TestAllTypesNano generateMessageForOneof(int caseNumber) { + TestAllTypesNano result = new TestAllTypesNano(); + TestAllTypesNano.NestedMessage nested = + new TestAllTypesNano.NestedMessage(); + nested.bb = 2; + switch (caseNumber) { + case TestAllTypesNano.ONEOF_UINT32_FIELD_NUMBER: + result.setOneofUint32(1); + break; + case TestAllTypesNano.ONEOF_ENUM_FIELD_NUMBER: + result.setOneofEnum(TestAllTypesNano.BAR); + break; + case TestAllTypesNano.ONEOF_NESTED_MESSAGE_FIELD_NUMBER: + result.setOneofNestedMessage(nested); + break; + case TestAllTypesNano.ONEOF_BYTES_FIELD_NUMBER: + result.setOneofBytes(new byte[] {1, 2}); + break; + case TestAllTypesNano.ONEOF_STRING_FIELD_NUMBER: + result.setOneofString("hello"); + break; + case TestAllTypesNano.ONEOF_FIXED64_FIELD_NUMBER: + result.setOneofFixed64(-1L); + break; + default: + throw new RuntimeException("unexpected case number: " + caseNumber); + } + return result; + } + + public void testOneofHashCodeEquals() throws Exception { + TestAllTypesNano m1 = generateMessageForOneof( + TestAllTypesNano.ONEOF_UINT32_FIELD_NUMBER); + assertEquals(m1, generateMessageForOneof( + TestAllTypesNano.ONEOF_UINT32_FIELD_NUMBER)); + assertFalse(m1.equals(new TestAllTypesNano())); + + TestAllTypesNano m2 = generateMessageForOneof( + TestAllTypesNano.ONEOF_ENUM_FIELD_NUMBER); + assertEquals(m2, generateMessageForOneof( + TestAllTypesNano.ONEOF_ENUM_FIELD_NUMBER)); + assertFalse(m2.equals(new TestAllTypesNano())); + + TestAllTypesNano m3 = generateMessageForOneof( + TestAllTypesNano.ONEOF_NESTED_MESSAGE_FIELD_NUMBER); + assertEquals(m3, generateMessageForOneof( + TestAllTypesNano.ONEOF_NESTED_MESSAGE_FIELD_NUMBER)); + assertFalse(m3.equals(new TestAllTypesNano())); + + TestAllTypesNano m4 = generateMessageForOneof( + TestAllTypesNano.ONEOF_BYTES_FIELD_NUMBER); + assertEquals(m4, generateMessageForOneof( + TestAllTypesNano.ONEOF_BYTES_FIELD_NUMBER)); + assertFalse(m4.equals(new TestAllTypesNano())); + + TestAllTypesNano m5 = generateMessageForOneof( + TestAllTypesNano.ONEOF_STRING_FIELD_NUMBER); + assertEquals(m5, generateMessageForOneof( + TestAllTypesNano.ONEOF_STRING_FIELD_NUMBER)); + assertFalse(m5.equals(new TestAllTypesNano())); + + TestAllTypesNano m6 = generateMessageForOneof( + TestAllTypesNano.ONEOF_FIXED64_FIELD_NUMBER); + assertEquals(m6, generateMessageForOneof( + TestAllTypesNano.ONEOF_FIXED64_FIELD_NUMBER)); + assertFalse(m6.equals(new TestAllTypesNano())); + + Map<TestAllTypesNano, Integer> map = + new HashMap<TestAllTypesNano, Integer>(); + map.put(m1, 1); + map.put(m2, 2); + map.put(m3, 3); + map.put(m4, 4); + map.put(m5, 5); + map.put(m6, 6); + + assertEquals(6, map.size()); + } + + private void checkOneofCase(TestAllTypesNano nano, int field) + throws Exception { + assertEquals(field, nano.getOneofFieldCase()); + assertEquals( + field == TestAllTypesNano.ONEOF_BYTES_FIELD_NUMBER, + nano.hasOneofBytes()); + assertEquals( + field == TestAllTypesNano.ONEOF_ENUM_FIELD_NUMBER, + nano.hasOneofEnum()); + assertEquals( + field == TestAllTypesNano.ONEOF_FIXED64_FIELD_NUMBER, + nano.hasOneofFixed64()); + assertEquals( + field == TestAllTypesNano.ONEOF_NESTED_MESSAGE_FIELD_NUMBER, + nano.hasOneofNestedMessage()); + assertEquals( + field == TestAllTypesNano.ONEOF_STRING_FIELD_NUMBER, + nano.hasOneofString()); + assertEquals( + field == TestAllTypesNano.ONEOF_UINT32_FIELD_NUMBER, + nano.hasOneofUint32()); + + } + + public void testOneofDefault() throws Exception { + TestAllTypesNano m1 = new TestAllTypesNano(); + checkOneofCase(m1, 0); + assertEquals(WireFormatNano.EMPTY_BYTES, m1.getOneofBytes()); + assertEquals(TestAllTypesNano.FOO, m1.getOneofEnum()); + assertEquals(0L, m1.getOneofFixed64()); + assertEquals(null, m1.getOneofNestedMessage()); + assertEquals("", m1.getOneofString()); + assertEquals(0, m1.getOneofUint32()); + } + + public void testOneofExclusiveness() throws Exception { + TestAllTypesNano m = new TestAllTypesNano(); + checkOneofCase(m, 0); + + m.setOneofBytes(new byte[]{0, 1}); + checkOneofCase(m, TestAllTypesNano.ONEOF_BYTES_FIELD_NUMBER); + assertTrue(Arrays.equals(new byte[]{0, 1}, m.getOneofBytes())); + + m.setOneofEnum(TestAllTypesNano.BAZ); + checkOneofCase(m, TestAllTypesNano.ONEOF_ENUM_FIELD_NUMBER); + assertEquals(TestAllTypesNano.BAZ, m.getOneofEnum()); + assertEquals(WireFormatNano.EMPTY_BYTES, m.getOneofBytes()); + + m.setOneofFixed64(-1L); + checkOneofCase(m, TestAllTypesNano.ONEOF_FIXED64_FIELD_NUMBER); + assertEquals(-1L, m.getOneofFixed64()); + assertEquals(TestAllTypesNano.FOO, m.getOneofEnum()); + + m.setOneofNestedMessage(new TestAllTypesNano.NestedMessage()); + checkOneofCase(m, TestAllTypesNano.ONEOF_NESTED_MESSAGE_FIELD_NUMBER); + assertEquals( + new TestAllTypesNano.NestedMessage(), m.getOneofNestedMessage()); + assertEquals(0L, m.getOneofFixed64()); + + m.setOneofString("hello"); + checkOneofCase(m, TestAllTypesNano.ONEOF_STRING_FIELD_NUMBER); + assertEquals("hello", m.getOneofString()); + assertNull(m.getOneofNestedMessage()); + + m.setOneofUint32(10); + checkOneofCase(m, TestAllTypesNano.ONEOF_UINT32_FIELD_NUMBER); + assertEquals(10, m.getOneofUint32()); + assertEquals("", m.getOneofString()); + + m.setOneofBytes(new byte[]{0, 1}); + checkOneofCase(m, TestAllTypesNano.ONEOF_BYTES_FIELD_NUMBER); + assertTrue(Arrays.equals(new byte[]{0, 1}, m.getOneofBytes())); + assertEquals(0, m.getOneofUint32()); + } + + public void testOneofClear() throws Exception { + TestAllTypesNano m = new TestAllTypesNano(); + m.setOneofBytes(new byte[]{0, 1}); + m.clearOneofField(); + checkOneofCase(m, 0); + + m.setOneofEnum(TestAllTypesNano.BAZ); + m.clearOneofField(); + checkOneofCase(m, 0); + + m.setOneofFixed64(-1L); + m.clearOneofField(); + checkOneofCase(m, 0); + + m.setOneofNestedMessage(new TestAllTypesNano.NestedMessage()); + m.clearOneofField(); + checkOneofCase(m, 0); + + m.setOneofString("hello"); + m.clearOneofField(); + checkOneofCase(m, 0); + + m.setOneofUint32(10); + m.clearOneofField(); + checkOneofCase(m, 0); + } + + public void testOneofMarshaling() throws Exception { + TestAllTypesNano m = new TestAllTypesNano(); + TestAllTypesNano parsed = new TestAllTypesNano(); + { + m.setOneofBytes(new byte[]{0, 1}); + byte[] serialized = MessageNano.toByteArray(m); + MessageNano.mergeFrom(parsed, serialized); + checkOneofCase(parsed, TestAllTypesNano.ONEOF_BYTES_FIELD_NUMBER); + assertTrue(Arrays.equals(new byte[]{0, 1}, parsed.getOneofBytes())); + } + { + m.setOneofEnum(TestAllTypesNano.BAZ); + byte[] serialized = MessageNano.toByteArray(m); + MessageNano.mergeFrom(parsed, serialized); + checkOneofCase(m, TestAllTypesNano.ONEOF_ENUM_FIELD_NUMBER); + assertEquals(TestAllTypesNano.BAZ, m.getOneofEnum()); + } + { + m.setOneofEnum(TestAllTypesNano.BAZ); + byte[] serialized = MessageNano.toByteArray(m); + MessageNano.mergeFrom(parsed, serialized); + checkOneofCase(m, TestAllTypesNano.ONEOF_ENUM_FIELD_NUMBER); + assertEquals(TestAllTypesNano.BAZ, m.getOneofEnum()); + } + { + m.setOneofFixed64(-1L); + byte[] serialized = MessageNano.toByteArray(m); + MessageNano.mergeFrom(parsed, serialized); + checkOneofCase(m, TestAllTypesNano.ONEOF_FIXED64_FIELD_NUMBER); + assertEquals(-1L, m.getOneofFixed64()); + } + { + m.setOneofNestedMessage(new TestAllTypesNano.NestedMessage()); + byte[] serialized = MessageNano.toByteArray(m); + MessageNano.mergeFrom(parsed, serialized); + checkOneofCase(m, TestAllTypesNano.ONEOF_NESTED_MESSAGE_FIELD_NUMBER); + assertEquals( + new TestAllTypesNano.NestedMessage(), m.getOneofNestedMessage()); + } + { + m.setOneofString("hello"); + byte[] serialized = MessageNano.toByteArray(m); + MessageNano.mergeFrom(parsed, serialized); + assertEquals("hello", m.getOneofString()); + assertNull(m.getOneofNestedMessage()); + } + { + m.setOneofUint32(10); + byte[] serialized = MessageNano.toByteArray(m); + MessageNano.mergeFrom(parsed, serialized); + checkOneofCase(m, TestAllTypesNano.ONEOF_UINT32_FIELD_NUMBER); + assertEquals(10, m.getOneofUint32()); + } + } + + public void testOneofSerializedConcat() throws Exception { + TestAllTypesNano m1 = new TestAllTypesNano(); + m1.setOneofBytes(new byte[] {0, 1}); + byte[] b1 = MessageNano.toByteArray(m1); + TestAllTypesNano m2 = new TestAllTypesNano(); + m2.setOneofEnum(TestAllTypesNano.BAZ); + byte[] b2 = MessageNano.toByteArray(m2); + byte[] b3 = new byte[b1.length + b2.length]; + System.arraycopy(b1, 0, b3, 0, b1.length); + System.arraycopy(b2, 0, b3, b1.length, b2.length); + TestAllTypesNano parsed = new TestAllTypesNano(); + MessageNano.mergeFrom(parsed, b3); + // the last on the wire wins. + checkOneofCase(parsed, TestAllTypesNano.ONEOF_ENUM_FIELD_NUMBER); + assertEquals(TestAllTypesNano.BAZ, parsed.getOneofEnum()); + } + public void testNullRepeatedFields() throws Exception { // Check that serialization after explicitly setting a repeated field // to null doesn't NPE. @@ -3754,6 +4031,320 @@ public class NanoTest extends TestCase { assertTrue(Arrays.equals(new boolean[] {false, true, false, true}, nonPacked.bools)); } + public void testMapsSerializeAndParse() throws Exception { + TestMap origin = new TestMap(); + setMapMessage(origin); + assertMapMessageSet(origin); + + byte[] output = MessageNano.toByteArray(origin); + TestMap parsed = new TestMap(); + MessageNano.mergeFrom(parsed, output); + } + + public void testMapSerializeRejectNull() throws Exception { + TestMap primitiveMap = new TestMap(); + primitiveMap.int32ToInt32Field = new HashMap<Integer, Integer>(); + primitiveMap.int32ToInt32Field.put(null, 1); + try { + MessageNano.toByteArray(primitiveMap); + fail("should reject null keys"); + } catch (IllegalStateException e) { + // pass. + } + + TestMap messageMap = new TestMap(); + messageMap.int32ToMessageField = + new HashMap<Integer, MapTestProto.TestMap.MessageValue>(); + messageMap.int32ToMessageField.put(0, null); + try { + MessageNano.toByteArray(messageMap); + fail("should reject null values"); + } catch (IllegalStateException e) { + // pass. + } + } + + /** + * Tests that merging bytes containing conflicting keys with override the + * message value instead of merging the message value into the existing entry. + */ + public void testMapMergeOverrideMessageValues() throws Exception { + TestMap.MessageValue origValue = new TestMap.MessageValue(); + origValue.value = 1; + origValue.value2 = 2; + TestMap.MessageValue newValue = new TestMap.MessageValue(); + newValue.value = 3; + + TestMap origMessage = new TestMap(); + origMessage.int32ToMessageField = + new HashMap<Integer, MapTestProto.TestMap.MessageValue>(); + origMessage.int32ToMessageField.put(1, origValue); + + TestMap newMessage = new TestMap(); + newMessage.int32ToMessageField = + new HashMap<Integer, MapTestProto.TestMap.MessageValue>(); + newMessage.int32ToMessageField.put(1, newValue); + MessageNano.mergeFrom(origMessage, + MessageNano.toByteArray(newMessage)); + TestMap.MessageValue mergedValue = origMessage.int32ToMessageField.get(1); + assertEquals(3, mergedValue.value); + assertEquals(0, mergedValue.value2); + } + + /** + * Tests that when merging with empty entries, + * we will use default for the key and value, instead of null. + */ + public void testMapMergeEmptyEntry() throws Exception { + TestMap testMap = new TestMap(); + byte[] buffer = new byte[1024]; + CodedOutputByteBufferNano output = + CodedOutputByteBufferNano.newInstance(buffer); + // An empty entry for int32_to_int32 map. + output.writeTag(1, WireFormatNano.WIRETYPE_LENGTH_DELIMITED); + output.writeRawVarint32(0); + // An empty entry for int32_to_message map. + output.writeTag(5, WireFormatNano.WIRETYPE_LENGTH_DELIMITED); + output.writeRawVarint32(0); + + CodedInputByteBufferNano input = CodedInputByteBufferNano.newInstance( + buffer, 0, buffer.length - output.spaceLeft()); + testMap.mergeFrom(input); + assertNotNull(testMap.int32ToInt32Field);; + assertEquals(1, testMap.int32ToInt32Field.size()); + assertEquals(Integer.valueOf(0), testMap.int32ToInt32Field.get(0)); + assertNotNull(testMap.int32ToMessageField); + assertEquals(1, testMap.int32ToMessageField.size()); + TestMap.MessageValue messageValue = testMap.int32ToMessageField.get(0); + assertNotNull(messageValue); + assertEquals(0, messageValue.value); + assertEquals(0, messageValue.value2); + } + + public void testMapEquals() throws Exception { + TestMap a = new TestMap(); + TestMap b = new TestMap(); + + // empty and null map fields are equal. + assertTestMapEqual(a, b); + a.int32ToBytesField = new HashMap<Integer, byte[]>(); + assertTestMapEqual(a, b); + + a.int32ToInt32Field = new HashMap<Integer, Integer>(); + b.int32ToInt32Field = new HashMap<Integer, Integer>(); + setMap(a.int32ToInt32Field, deepCopy(int32Values), deepCopy(int32Values)); + setMap(b.int32ToInt32Field, deepCopy(int32Values), deepCopy(int32Values)); + assertTestMapEqual(a, b); + + a.int32ToMessageField = + new HashMap<Integer, MapTestProto.TestMap.MessageValue>(); + b.int32ToMessageField = + new HashMap<Integer, MapTestProto.TestMap.MessageValue>(); + setMap(a.int32ToMessageField, + deepCopy(int32Values), deepCopy(messageValues)); + setMap(b.int32ToMessageField, + deepCopy(int32Values), deepCopy(messageValues)); + assertTestMapEqual(a, b); + + a.stringToInt32Field = new HashMap<String, Integer>(); + b.stringToInt32Field = new HashMap<String, Integer>(); + setMap(a.stringToInt32Field, deepCopy(stringValues), deepCopy(int32Values)); + setMap(b.stringToInt32Field, deepCopy(stringValues), deepCopy(int32Values)); + assertTestMapEqual(a, b); + + a.int32ToBytesField = new HashMap<Integer, byte[]>(); + b.int32ToBytesField = new HashMap<Integer, byte[]>(); + setMap(a.int32ToBytesField, deepCopy(int32Values), deepCopy(bytesValues)); + setMap(b.int32ToBytesField, deepCopy(int32Values), deepCopy(bytesValues)); + assertTestMapEqual(a, b); + + // Make sure the map implementation does not matter. + a.int32ToStringField = new TreeMap<Integer, String>(); + b.int32ToStringField = new HashMap<Integer, String>(); + setMap(a.int32ToStringField, deepCopy(int32Values), deepCopy(stringValues)); + setMap(b.int32ToStringField, deepCopy(int32Values), deepCopy(stringValues)); + assertTestMapEqual(a, b); + + a.clear(); + b.clear(); + + // unequal cases: different value + a.int32ToInt32Field = new HashMap<Integer, Integer>(); + b.int32ToInt32Field = new HashMap<Integer, Integer>(); + a.int32ToInt32Field.put(1, 1); + b.int32ToInt32Field.put(1, 2); + assertTestMapUnequal(a, b); + // unequal case: additional entry + b.int32ToInt32Field.put(1, 1); + b.int32ToInt32Field.put(2, 1); + assertTestMapUnequal(a, b); + a.int32ToInt32Field.put(2, 1); + assertTestMapEqual(a, b); + + // unequal case: different message value. + a.int32ToMessageField = + new HashMap<Integer, MapTestProto.TestMap.MessageValue>(); + b.int32ToMessageField = + new HashMap<Integer, MapTestProto.TestMap.MessageValue>(); + MessageValue va = new MessageValue(); + va.value = 1; + MessageValue vb = new MessageValue(); + vb.value = 1; + a.int32ToMessageField.put(1, va); + b.int32ToMessageField.put(1, vb); + assertTestMapEqual(a, b); + vb.value = 2; + assertTestMapUnequal(a, b); + } + + private static void assertTestMapEqual(TestMap a, TestMap b) + throws Exception { + assertEquals(a.hashCode(), b.hashCode()); + assertTrue(a.equals(b)); + assertTrue(b.equals(a)); + } + + private static void assertTestMapUnequal(TestMap a, TestMap b) + throws Exception { + assertFalse(a.equals(b)); + assertFalse(b.equals(a)); + } + + private static final Integer[] int32Values = new Integer[] { + 0, 1, -1, Integer.MAX_VALUE, Integer.MIN_VALUE, + }; + + private static final Long[] int64Values = new Long[] { + 0L, 1L, -1L, Long.MAX_VALUE, Long.MIN_VALUE, + }; + + private static final String[] stringValues = new String[] { + "", "hello", "world", "foo", "bar", + }; + + private static final byte[][] bytesValues = new byte[][] { + new byte[] {}, + new byte[] {0}, + new byte[] {1, -1}, + new byte[] {127, -128}, + new byte[] {'a', 'b', '0', '1'}, + }; + + private static final Boolean[] boolValues = new Boolean[] { + false, true, + }; + + private static final Integer[] enumValues = new Integer[] { + TestMap.FOO, TestMap.BAR, TestMap.BAZ, TestMap.QUX, + Integer.MAX_VALUE /* unknown */, + }; + + private static final TestMap.MessageValue[] messageValues = + new TestMap.MessageValue[] { + newMapValueMessage(0), + newMapValueMessage(1), + newMapValueMessage(-1), + newMapValueMessage(Integer.MAX_VALUE), + newMapValueMessage(Integer.MIN_VALUE), + }; + + private static TestMap.MessageValue newMapValueMessage(int value) { + TestMap.MessageValue result = new TestMap.MessageValue(); + result.value = value; + return result; + } + + @SuppressWarnings("unchecked") + private static <T> T[] deepCopy(T[] orig) throws Exception { + if (orig instanceof MessageValue[]) { + MessageValue[] result = new MessageValue[orig.length]; + for (int i = 0; i < orig.length; i++) { + result[i] = new MessageValue(); + MessageNano.mergeFrom( + result[i], MessageNano.toByteArray((MessageValue) orig[i])); + } + return (T[]) result; + } + if (orig instanceof byte[][]) { + byte[][] result = new byte[orig.length][]; + for (int i = 0; i < orig.length; i++) { + byte[] origBytes = (byte[]) orig[i]; + result[i] = Arrays.copyOf(origBytes, origBytes.length); + } + } + return Arrays.copyOf(orig, orig.length); + } + + private <K, V> void setMap(Map<K, V> map, K[] keys, V[] values) { + assert(keys.length == values.length); + for (int i = 0; i < keys.length; i++) { + map.put(keys[i], values[i]); + } + } + + private <K, V> void assertMapSet( + Map<K, V> map, K[] keys, V[] values) throws Exception { + assert(keys.length == values.length); + for (int i = 0; i < values.length; i++) { + assertEquals(values[i], map.get(keys[i])); + } + assertEquals(keys.length, map.size()); + } + + private void setMapMessage(TestMap testMap) { + testMap.int32ToInt32Field = new HashMap<Integer, Integer>(); + testMap.int32ToBytesField = new HashMap<Integer, byte[]>(); + testMap.int32ToEnumField = new HashMap<Integer, Integer>(); + testMap.int32ToMessageField = + new HashMap<Integer, MapTestProto.TestMap.MessageValue>(); + testMap.int32ToStringField = new HashMap<Integer, String>(); + testMap.stringToInt32Field = new HashMap<String, Integer>(); + testMap.boolToBoolField = new HashMap<Boolean, Boolean>(); + testMap.uint32ToUint32Field = new HashMap<Integer, Integer>(); + testMap.sint32ToSint32Field = new HashMap<Integer, Integer>(); + testMap.fixed32ToFixed32Field = new HashMap<Integer, Integer>(); + testMap.sfixed32ToSfixed32Field = new HashMap<Integer, Integer>(); + testMap.int64ToInt64Field = new HashMap<Long, Long>(); + testMap.uint64ToUint64Field = new HashMap<Long, Long>(); + testMap.sint64ToSint64Field = new HashMap<Long, Long>(); + testMap.fixed64ToFixed64Field = new HashMap<Long, Long>(); + testMap.sfixed64ToSfixed64Field = new HashMap<Long, Long>(); + setMap(testMap.int32ToInt32Field, int32Values, int32Values); + setMap(testMap.int32ToBytesField, int32Values, bytesValues); + setMap(testMap.int32ToEnumField, int32Values, enumValues); + setMap(testMap.int32ToMessageField, int32Values, messageValues); + setMap(testMap.int32ToStringField, int32Values, stringValues); + setMap(testMap.stringToInt32Field, stringValues, int32Values); + setMap(testMap.boolToBoolField, boolValues, boolValues); + setMap(testMap.uint32ToUint32Field, int32Values, int32Values); + setMap(testMap.sint32ToSint32Field, int32Values, int32Values); + setMap(testMap.fixed32ToFixed32Field, int32Values, int32Values); + setMap(testMap.sfixed32ToSfixed32Field, int32Values, int32Values); + setMap(testMap.int64ToInt64Field, int64Values, int64Values); + setMap(testMap.uint64ToUint64Field, int64Values, int64Values); + setMap(testMap.sint64ToSint64Field, int64Values, int64Values); + setMap(testMap.fixed64ToFixed64Field, int64Values, int64Values); + setMap(testMap.sfixed64ToSfixed64Field, int64Values, int64Values); + } + private void assertMapMessageSet(TestMap testMap) throws Exception { + assertMapSet(testMap.int32ToInt32Field, int32Values, int32Values); + assertMapSet(testMap.int32ToBytesField, int32Values, bytesValues); + assertMapSet(testMap.int32ToEnumField, int32Values, enumValues); + assertMapSet(testMap.int32ToMessageField, int32Values, messageValues); + assertMapSet(testMap.int32ToStringField, int32Values, stringValues); + assertMapSet(testMap.stringToInt32Field, stringValues, int32Values); + assertMapSet(testMap.boolToBoolField, boolValues, boolValues); + assertMapSet(testMap.uint32ToUint32Field, int32Values, int32Values); + assertMapSet(testMap.sint32ToSint32Field, int32Values, int32Values); + assertMapSet(testMap.fixed32ToFixed32Field, int32Values, int32Values); + assertMapSet(testMap.sfixed32ToSfixed32Field, int32Values, int32Values); + assertMapSet(testMap.int64ToInt64Field, int64Values, int64Values); + assertMapSet(testMap.uint64ToUint64Field, int64Values, int64Values); + assertMapSet(testMap.sint64ToSint64Field, int64Values, int64Values); + assertMapSet(testMap.fixed64ToFixed64Field, int64Values, int64Values); + assertMapSet(testMap.sfixed64ToSfixed64Field, int64Values, int64Values); + } + private void assertRepeatedPackablesEqual( NanoRepeatedPackables.NonPacked nonPacked, NanoRepeatedPackables.Packed packed) { // Not using MessageNano.equals() -- that belongs to a separate test. diff --git a/javanano/src/test/java/com/google/protobuf/nano/map_test.proto b/javanano/src/test/java/com/google/protobuf/nano/map_test.proto new file mode 100644 index 00000000..f72833ad --- /dev/null +++ b/javanano/src/test/java/com/google/protobuf/nano/map_test.proto @@ -0,0 +1,70 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package map_test; + +option java_package = "com.google.protobuf.nano"; +option java_outer_classname = "MapTestProto"; + +message TestMap { + message MessageValue { + int32 value = 1; + int32 value2 = 2; + } + enum EnumValue { + FOO = 0; + BAR = 1; + BAZ = 2; + QUX = 3; + } + + map<int32, int32> int32_to_int32_field = 1; + map<int32, string> int32_to_string_field = 2; + map<int32, bytes> int32_to_bytes_field = 3; + map<int32, EnumValue> int32_to_enum_field = 4; + map<int32, MessageValue> int32_to_message_field = 5; + map<string, int32> string_to_int32_field = 6; + map<bool, bool> bool_to_bool_field = 7; + + // Test all the other primitive types. As the key and value are not coupled in + // the implementation, we do not test all the combinations of key/value pairs, + // so that we can keep the number of test cases manageable + map<uint32, uint32> uint32_to_uint32_field = 11; + map<sint32, sint32> sint32_to_sint32_field = 12; + map<fixed32, fixed32> fixed32_to_fixed32_field = 13; + map<sfixed32, sfixed32> sfixed32_to_sfixed32_field = 14; + map<int64, int64> int64_to_int64_field = 15; + map<uint64, uint64> uint64_to_uint64_field = 16; + map<sint64, sint64> sint64_to_sint64_field = 17; + map<fixed64, fixed64> fixed64_to_fixed64_field = 18; + map<sfixed64, sfixed64> sfixed64_to_sfixed64_field = 19; +} diff --git a/javanano/src/test/java/com/google/protobuf/nano/unittest_nano.proto b/javanano/src/test/java/com/google/protobuf/nano/unittest_nano.proto index 4519533c..efaf5269 100644 --- a/javanano/src/test/java/com/google/protobuf/nano/unittest_nano.proto +++ b/javanano/src/test/java/com/google/protobuf/nano/unittest_nano.proto @@ -167,6 +167,15 @@ message TestAllTypesNano { // Try to fail with java reserved keywords optional int32 synchronized = 96; + + oneof oneof_field { + uint32 oneof_uint32 = 111; + NestedMessage oneof_nested_message = 112; + string oneof_string = 123; + bytes oneof_bytes = 124; + fixed64 oneof_fixed64 = 115; + NestedEnum oneof_enum = 116; + } } message ForeignMessageNano { diff --git a/post_process_dist.sh b/post_process_dist.sh index 733fa088..3c01ed8f 100755 --- a/post_process_dist.sh +++ b/post_process_dist.sh @@ -27,7 +27,7 @@ fi set -ex -LANGUAGES="cpp java python" +LANGUAGES="cpp java javanano python ruby" BASENAME=`basename $1 .tar.gz` VERSION=${BASENAME:9} diff --git a/python/README.txt b/python/README.txt index 7d852458..04cb1767 100644 --- a/python/README.txt +++ b/python/README.txt @@ -48,8 +48,9 @@ Installation $ python setup.py build $ python setup.py google_test - If you want to test c++ implementation, run: - $ python setup.py test --cpp_implementation + If you want to build/test c++ implementation, run: + $ python setup.py build --cpp_implementation + $ python setup.py google_test --cpp_implementation If some tests fail, this library may not work correctly on your system. Continue at your own risk. diff --git a/python/google/protobuf/descriptor.py b/python/google/protobuf/descriptor.py index af571b7c..f7a58ca0 100755 --- a/python/google/protobuf/descriptor.py +++ b/python/google/protobuf/descriptor.py @@ -41,15 +41,13 @@ __author__ = 'robinson@google.com (Will Robinson)' from google.protobuf.internal import api_implementation +_USE_C_DESCRIPTORS = False if api_implementation.Type() == 'cpp': # Used by MakeDescriptor in cpp mode import os import uuid - - if api_implementation.Version() == 2: - from google.protobuf.pyext import _message - else: - from google.protobuf.internal import cpp_message + from google.protobuf.pyext import _message + _USE_C_DESCRIPTORS = getattr(_message, '_USE_C_DESCRIPTORS', False) class Error(Exception): @@ -60,12 +58,29 @@ class TypeTransformationError(Error): """Error transforming between python proto type and corresponding C++ type.""" +if _USE_C_DESCRIPTORS: + # This metaclass allows to override the behavior of code like + # isinstance(my_descriptor, FieldDescriptor) + # and make it return True when the descriptor is an instance of the extension + # type written in C++. + class DescriptorMetaclass(type): + def __instancecheck__(cls, obj): + if super(DescriptorMetaclass, cls).__instancecheck__(obj): + return True + if isinstance(obj, cls._C_DESCRIPTOR_CLASS): + return True + return False +else: + # The standard metaclass; nothing changes. + DescriptorMetaclass = type + + class DescriptorBase(object): """Descriptors base class. This class is the base of all descriptor classes. It provides common options - related functionaility. + related functionality. Attributes: has_options: True if the descriptor has non-default options. Usually it @@ -75,6 +90,12 @@ class DescriptorBase(object): avoid some bootstrapping issues. """ + __metaclass__ = DescriptorMetaclass + if _USE_C_DESCRIPTORS: + # The class, or tuple of classes, that are considered as "virtual + # subclasses" of this descriptor class. + _C_DESCRIPTOR_CLASS = () + def __init__(self, options, options_class_name): """Initialize the descriptor given its options message and the name of the class of the options message. The name of the class is required in case @@ -235,13 +256,25 @@ class Descriptor(_NestedDescriptorBase): file: (FileDescriptor) Reference to file descriptor. """ + if _USE_C_DESCRIPTORS: + _C_DESCRIPTOR_CLASS = _message.Descriptor + + def __new__(cls, name, full_name, filename, containing_type, fields, + nested_types, enum_types, extensions, options=None, + is_extendable=True, extension_ranges=None, oneofs=None, + file=None, serialized_start=None, serialized_end=None, + syntax=None): + _message.Message._CheckCalledFromGeneratedFile() + return _message.Message._GetMessageDescriptor(full_name) + # NOTE(tmarek): The file argument redefining a builtin is nothing we can # fix right now since we don't know how many clients already rely on the # name of the argument. def __init__(self, name, full_name, filename, containing_type, fields, nested_types, enum_types, extensions, options=None, is_extendable=True, extension_ranges=None, oneofs=None, - file=None, serialized_start=None, serialized_end=None): # pylint:disable=redefined-builtin + file=None, serialized_start=None, serialized_end=None, + syntax=None): # pylint:disable=redefined-builtin """Arguments to __init__() are as described in the description of Descriptor fields above. @@ -286,6 +319,7 @@ class Descriptor(_NestedDescriptorBase): self.oneofs_by_name = dict((o.name, o) for o in self.oneofs) for oneof in self.oneofs: oneof.containing_type = self + self.syntax = syntax or "proto2" def EnumValueName(self, enum, value): """Returns the string name of an enum value. @@ -452,6 +486,19 @@ class FieldDescriptor(DescriptorBase): FIRST_RESERVED_FIELD_NUMBER = 19000 LAST_RESERVED_FIELD_NUMBER = 19999 + if _USE_C_DESCRIPTORS: + _C_DESCRIPTOR_CLASS = _message.FieldDescriptor + + def __new__(cls, name, full_name, index, number, type, cpp_type, label, + default_value, message_type, enum_type, containing_type, + is_extension, extension_scope, options=None, + has_default_value=True, containing_oneof=None): + _message.Message._CheckCalledFromGeneratedFile() + if is_extension: + return _message.Message._GetExtensionDescriptor(full_name) + else: + return _message.Message._GetFieldDescriptor(full_name) + def __init__(self, name, full_name, index, number, type, cpp_type, label, default_value, message_type, enum_type, containing_type, is_extension, extension_scope, options=None, @@ -481,20 +528,14 @@ class FieldDescriptor(DescriptorBase): self.containing_oneof = containing_oneof if api_implementation.Type() == 'cpp': if is_extension: - if api_implementation.Version() == 2: - # pylint: disable=protected-access - self._cdescriptor = ( - _message.Message._GetExtensionDescriptor(full_name)) - # pylint: enable=protected-access - else: - self._cdescriptor = cpp_message.GetExtensionDescriptor(full_name) + # pylint: disable=protected-access + self._cdescriptor = ( + _message.Message._GetExtensionDescriptor(full_name)) + # pylint: enable=protected-access else: - if api_implementation.Version() == 2: - # pylint: disable=protected-access - self._cdescriptor = _message.Message._GetFieldDescriptor(full_name) - # pylint: enable=protected-access - else: - self._cdescriptor = cpp_message.GetFieldDescriptor(full_name) + # pylint: disable=protected-access + self._cdescriptor = _message.Message._GetFieldDescriptor(full_name) + # pylint: enable=protected-access else: self._cdescriptor = None @@ -544,6 +585,15 @@ class EnumDescriptor(_NestedDescriptorBase): None to use default enum options. """ + if _USE_C_DESCRIPTORS: + _C_DESCRIPTOR_CLASS = _message.EnumDescriptor + + def __new__(cls, name, full_name, filename, values, + containing_type=None, options=None, file=None, + serialized_start=None, serialized_end=None): + _message.Message._CheckCalledFromGeneratedFile() + return _message.Message._GetEnumDescriptor(full_name) + def __init__(self, name, full_name, filename, values, containing_type=None, options=None, file=None, serialized_start=None, serialized_end=None): @@ -588,6 +638,17 @@ class EnumValueDescriptor(DescriptorBase): None to use default enum value options options. """ + if _USE_C_DESCRIPTORS: + _C_DESCRIPTOR_CLASS = _message.EnumValueDescriptor + + def __new__(cls, name, index, number, type=None, options=None): + _message.Message._CheckCalledFromGeneratedFile() + # There is no way we can build a complete EnumValueDescriptor with the + # given parameters (the name of the Enum is not known, for example). + # Fortunately generated files just pass it to the EnumDescriptor() + # constructor, which will ignore it, so returning None is good enough. + return None + def __init__(self, name, index, number, type=None, options=None): """Arguments are as described in the attribute description above.""" super(EnumValueDescriptor, self).__init__(options, 'EnumValueOptions') @@ -611,6 +672,13 @@ class OneofDescriptor(object): oneof can contain. """ + if _USE_C_DESCRIPTORS: + _C_DESCRIPTOR_CLASS = _message.OneofDescriptor + + def __new__(cls, name, full_name, index, containing_type, fields): + _message.Message._CheckCalledFromGeneratedFile() + return _message.Message._GetOneofDescriptor(full_name) + def __init__(self, name, full_name, index, containing_type, fields): """Arguments are as described in the attribute description above.""" self.name = name @@ -704,6 +772,7 @@ class FileDescriptor(DescriptorBase): name: name of file, relative to root of source tree. package: name of the package + syntax: string indicating syntax of the file (can be "proto2" or "proto3") serialized_pb: (str) Byte string of serialized descriptor_pb2.FileDescriptorProto. dependencies: List of other FileDescriptors this FileDescriptor depends on. @@ -712,14 +781,31 @@ class FileDescriptor(DescriptorBase): extensions_by_name: Dict of extension names and their descriptors. """ + if _USE_C_DESCRIPTORS: + _C_DESCRIPTOR_CLASS = _message.FileDescriptor + + def __new__(cls, name, package, options=None, serialized_pb=None, + dependencies=None, syntax=None): + # FileDescriptor() is called from various places, not only from generated + # files, to register dynamic proto files and messages. + # TODO(amauryfa): Expose BuildFile() as a public function and make this + # constructor an implementation detail. + if serialized_pb: + # pylint: disable=protected-access2 + return _message.Message._BuildFile(serialized_pb) + # pylint: enable=protected-access + else: + return super(FileDescriptor, cls).__new__(cls) + def __init__(self, name, package, options=None, serialized_pb=None, - dependencies=None): + dependencies=None, syntax=None): """Constructor.""" super(FileDescriptor, self).__init__(options, 'FileOptions') self.message_types_by_name = {} self.name = name self.package = package + self.syntax = syntax or "proto2" self.serialized_pb = serialized_pb self.enum_types_by_name = {} @@ -728,12 +814,9 @@ class FileDescriptor(DescriptorBase): if (api_implementation.Type() == 'cpp' and self.serialized_pb is not None): - if api_implementation.Version() == 2: - # pylint: disable=protected-access - _message.Message._BuildFile(self.serialized_pb) - # pylint: enable=protected-access - else: - cpp_message.BuildFile(self.serialized_pb) + # pylint: disable=protected-access + _message.Message._BuildFile(self.serialized_pb) + # pylint: enable=protected-access def CopyToProto(self, proto): """Copies this to a descriptor_pb2.FileDescriptorProto. @@ -754,7 +837,8 @@ def _ParseOptions(message, string): return message -def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True): +def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True, + syntax=None): """Make a protobuf Descriptor given a DescriptorProto protobuf. Handles nested descriptors. Note that this is limited to the scope of defining @@ -766,6 +850,8 @@ def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True): package: Optional package name for the new message Descriptor (string). build_file_if_cpp: Update the C++ descriptor pool if api matches. Set to False on recursion, so no duplicates are created. + syntax: The syntax/semantics that should be used. Set to "proto3" to get + proto3 field presence semantics. Returns: A Descriptor for protobuf messages. """ @@ -791,12 +877,13 @@ def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True): else: file_descriptor_proto.name = proto_name + '.proto' - if api_implementation.Version() == 2: - # pylint: disable=protected-access - _message.Message._BuildFile(file_descriptor_proto.SerializeToString()) - # pylint: enable=protected-access - else: - cpp_message.BuildFile(file_descriptor_proto.SerializeToString()) + # pylint: disable=protected-access + result = _message.Message._BuildFile( + file_descriptor_proto.SerializeToString()) + # pylint: enable=protected-access + + if _USE_C_DESCRIPTORS: + return result.message_types_by_name[desc_proto.name] full_message_name = [desc_proto.name] if package: full_message_name.insert(0, package) @@ -819,7 +906,8 @@ def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True): # used by fields in the message, so no loops are possible here. nested_desc = MakeDescriptor(nested_proto, package='.'.join(full_message_name), - build_file_if_cpp=False) + build_file_if_cpp=False, + syntax=syntax) nested_types[full_name] = nested_desc fields = [] diff --git a/python/google/protobuf/descriptor_pool.py b/python/google/protobuf/descriptor_pool.py index bcac513a..7e7701f8 100644 --- a/python/google/protobuf/descriptor_pool.py +++ b/python/google/protobuf/descriptor_pool.py @@ -64,6 +64,9 @@ from google.protobuf import descriptor_database from google.protobuf import text_encoding +_USE_C_DESCRIPTORS = descriptor._USE_C_DESCRIPTORS + + def _NormalizeFullyQualifiedName(name): """Remove leading period from fully-qualified type name. @@ -271,58 +274,81 @@ class DescriptorPool(object): file_descriptor = descriptor.FileDescriptor( name=file_proto.name, package=file_proto.package, + syntax=file_proto.syntax, options=file_proto.options, serialized_pb=file_proto.SerializeToString(), dependencies=direct_deps) - scope = {} - - # This loop extracts all the message and enum types from all the - # dependencoes of the file_proto. This is necessary to create the - # scope of available message types when defining the passed in - # file proto. - for dependency in built_deps: - scope.update(self._ExtractSymbols( - dependency.message_types_by_name.values())) - scope.update((_PrefixWithDot(enum.full_name), enum) - for enum in dependency.enum_types_by_name.values()) - - for message_type in file_proto.message_type: - message_desc = self._ConvertMessageDescriptor( - message_type, file_proto.package, file_descriptor, scope) - file_descriptor.message_types_by_name[message_desc.name] = message_desc - - for enum_type in file_proto.enum_type: - file_descriptor.enum_types_by_name[enum_type.name] = ( - self._ConvertEnumDescriptor(enum_type, file_proto.package, - file_descriptor, None, scope)) - - for index, extension_proto in enumerate(file_proto.extension): - extension_desc = self.MakeFieldDescriptor( - extension_proto, file_proto.package, index, is_extension=True) - extension_desc.containing_type = self._GetTypeFromScope( - file_descriptor.package, extension_proto.extendee, scope) - self.SetFieldType(extension_proto, extension_desc, - file_descriptor.package, scope) - file_descriptor.extensions_by_name[extension_desc.name] = extension_desc - - for desc_proto in file_proto.message_type: - self.SetAllFieldTypes(file_proto.package, desc_proto, scope) - - if file_proto.package: - desc_proto_prefix = _PrefixWithDot(file_proto.package) + if _USE_C_DESCRIPTORS: + # When using C++ descriptors, all objects defined in the file were added + # to the C++ database when the FileDescriptor was built above. + # Just add them to this descriptor pool. + def _AddMessageDescriptor(message_desc): + self._descriptors[message_desc.full_name] = message_desc + for nested in message_desc.nested_types: + _AddMessageDescriptor(nested) + for enum_type in message_desc.enum_types: + _AddEnumDescriptor(enum_type) + def _AddEnumDescriptor(enum_desc): + self._enum_descriptors[enum_desc.full_name] = enum_desc + for message_type in file_descriptor.message_types_by_name.values(): + _AddMessageDescriptor(message_type) + for enum_type in file_descriptor.enum_types_by_name.values(): + _AddEnumDescriptor(enum_type) else: - desc_proto_prefix = '' + scope = {} + + # This loop extracts all the message and enum types from all the + # dependencies of the file_proto. This is necessary to create the + # scope of available message types when defining the passed in + # file proto. + for dependency in built_deps: + scope.update(self._ExtractSymbols( + dependency.message_types_by_name.values())) + scope.update((_PrefixWithDot(enum.full_name), enum) + for enum in dependency.enum_types_by_name.values()) + + for message_type in file_proto.message_type: + message_desc = self._ConvertMessageDescriptor( + message_type, file_proto.package, file_descriptor, scope, + file_proto.syntax) + file_descriptor.message_types_by_name[message_desc.name] = ( + message_desc) + + for enum_type in file_proto.enum_type: + file_descriptor.enum_types_by_name[enum_type.name] = ( + self._ConvertEnumDescriptor(enum_type, file_proto.package, + file_descriptor, None, scope)) + + for index, extension_proto in enumerate(file_proto.extension): + extension_desc = self.MakeFieldDescriptor( + extension_proto, file_proto.package, index, is_extension=True) + extension_desc.containing_type = self._GetTypeFromScope( + file_descriptor.package, extension_proto.extendee, scope) + self.SetFieldType(extension_proto, extension_desc, + file_descriptor.package, scope) + file_descriptor.extensions_by_name[extension_desc.name] = ( + extension_desc) + + for desc_proto in file_proto.message_type: + self.SetAllFieldTypes(file_proto.package, desc_proto, scope) + + if file_proto.package: + desc_proto_prefix = _PrefixWithDot(file_proto.package) + else: + desc_proto_prefix = '' + + for desc_proto in file_proto.message_type: + desc = self._GetTypeFromScope( + desc_proto_prefix, desc_proto.name, scope) + file_descriptor.message_types_by_name[desc_proto.name] = desc - for desc_proto in file_proto.message_type: - desc = self._GetTypeFromScope(desc_proto_prefix, desc_proto.name, scope) - file_descriptor.message_types_by_name[desc_proto.name] = desc self.Add(file_proto) self._file_descriptors[file_proto.name] = file_descriptor return self._file_descriptors[file_proto.name] def _ConvertMessageDescriptor(self, desc_proto, package=None, file_desc=None, - scope=None): + scope=None, syntax=None): """Adds the proto to the pool in the specified package. Args: @@ -349,7 +375,8 @@ class DescriptorPool(object): scope = {} nested = [ - self._ConvertMessageDescriptor(nested, desc_name, file_desc, scope) + self._ConvertMessageDescriptor( + nested, desc_name, file_desc, scope, syntax) for nested in desc_proto.nested_type] enums = [ self._ConvertEnumDescriptor(enum, desc_name, file_desc, None, scope) @@ -383,7 +410,8 @@ class DescriptorPool(object): extension_ranges=extension_ranges, file=file_desc, serialized_start=None, - serialized_end=None) + serialized_end=None, + syntax=syntax) for nested in desc.nested_types: nested.containing_type = desc for enum in desc.enum_types: diff --git a/python/google/protobuf/internal/_parameterized.py b/python/google/protobuf/internal/_parameterized.py new file mode 100755 index 00000000..6ed23308 --- /dev/null +++ b/python/google/protobuf/internal/_parameterized.py @@ -0,0 +1,438 @@ +#! /usr/bin/python +# +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Adds support for parameterized tests to Python's unittest TestCase class. + +A parameterized test is a method in a test case that is invoked with different +argument tuples. + +A simple example: + + class AdditionExample(parameterized.ParameterizedTestCase): + @parameterized.Parameters( + (1, 2, 3), + (4, 5, 9), + (1, 1, 3)) + def testAddition(self, op1, op2, result): + self.assertEquals(result, op1 + op2) + + +Each invocation is a separate test case and properly isolated just +like a normal test method, with its own setUp/tearDown cycle. In the +example above, there are three separate testcases, one of which will +fail due to an assertion error (1 + 1 != 3). + +Parameters for invididual test cases can be tuples (with positional parameters) +or dictionaries (with named parameters): + + class AdditionExample(parameterized.ParameterizedTestCase): + @parameterized.Parameters( + {'op1': 1, 'op2': 2, 'result': 3}, + {'op1': 4, 'op2': 5, 'result': 9}, + ) + def testAddition(self, op1, op2, result): + self.assertEquals(result, op1 + op2) + +If a parameterized test fails, the error message will show the +original test name (which is modified internally) and the arguments +for the specific invocation, which are part of the string returned by +the shortDescription() method on test cases. + +The id method of the test, used internally by the unittest framework, +is also modified to show the arguments. To make sure that test names +stay the same across several invocations, object representations like + + >>> class Foo(object): + ... pass + >>> repr(Foo()) + '<__main__.Foo object at 0x23d8610>' + +are turned into '<__main__.Foo>'. For even more descriptive names, +especially in test logs, you can use the NamedParameters decorator. In +this case, only tuples are supported, and the first parameters has to +be a string (or an object that returns an apt name when converted via +str()): + + class NamedExample(parameterized.ParameterizedTestCase): + @parameterized.NamedParameters( + ('Normal', 'aa', 'aaa', True), + ('EmptyPrefix', '', 'abc', True), + ('BothEmpty', '', '', True)) + def testStartsWith(self, prefix, string, result): + self.assertEquals(result, strings.startswith(prefix)) + +Named tests also have the benefit that they can be run individually +from the command line: + + $ testmodule.py NamedExample.testStartsWithNormal + . + -------------------------------------------------------------------- + Ran 1 test in 0.000s + + OK + +Parameterized Classes +===================== +If invocation arguments are shared across test methods in a single +ParameterizedTestCase class, instead of decorating all test methods +individually, the class itself can be decorated: + + @parameterized.Parameters( + (1, 2, 3) + (4, 5, 9)) + class ArithmeticTest(parameterized.ParameterizedTestCase): + def testAdd(self, arg1, arg2, result): + self.assertEqual(arg1 + arg2, result) + + def testSubtract(self, arg2, arg2, result): + self.assertEqual(result - arg1, arg2) + +Inputs from Iterables +===================== +If parameters should be shared across several test cases, or are dynamically +created from other sources, a single non-tuple iterable can be passed into +the decorator. This iterable will be used to obtain the test cases: + + class AdditionExample(parameterized.ParameterizedTestCase): + @parameterized.Parameters( + c.op1, c.op2, c.result for c in testcases + ) + def testAddition(self, op1, op2, result): + self.assertEquals(result, op1 + op2) + + +Single-Argument Test Methods +============================ +If a test method takes only one argument, the single argument does not need to +be wrapped into a tuple: + + class NegativeNumberExample(parameterized.ParameterizedTestCase): + @parameterized.Parameters( + -1, -3, -4, -5 + ) + def testIsNegative(self, arg): + self.assertTrue(IsNegative(arg)) +""" + +__author__ = 'tmarek@google.com (Torsten Marek)' + +import collections +import functools +import re +import types +import unittest +import uuid + +from google.apputils import basetest + +ADDR_RE = re.compile(r'\<([a-zA-Z0-9_\-\.]+) object at 0x[a-fA-F0-9]+\>') +_SEPARATOR = uuid.uuid1().hex +_FIRST_ARG = object() +_ARGUMENT_REPR = object() + + +def _CleanRepr(obj): + return ADDR_RE.sub(r'<\1>', repr(obj)) + + +# Helper function formerly from the unittest module, removed from it in +# Python 2.7. +def _StrClass(cls): + return '%s.%s' % (cls.__module__, cls.__name__) + + +def _NonStringIterable(obj): + return (isinstance(obj, collections.Iterable) and not + isinstance(obj, basestring)) + + +def _FormatParameterList(testcase_params): + if isinstance(testcase_params, collections.Mapping): + return ', '.join('%s=%s' % (argname, _CleanRepr(value)) + for argname, value in testcase_params.iteritems()) + elif _NonStringIterable(testcase_params): + return ', '.join(map(_CleanRepr, testcase_params)) + else: + return _FormatParameterList((testcase_params,)) + + +class _ParameterizedTestIter(object): + """Callable and iterable class for producing new test cases.""" + + def __init__(self, test_method, testcases, naming_type): + """Returns concrete test functions for a test and a list of parameters. + + The naming_type is used to determine the name of the concrete + functions as reported by the unittest framework. If naming_type is + _FIRST_ARG, the testcases must be tuples, and the first element must + have a string representation that is a valid Python identifier. + + Args: + test_method: The decorated test method. + testcases: (list of tuple/dict) A list of parameter + tuples/dicts for individual test invocations. + naming_type: The test naming type, either _NAMED or _ARGUMENT_REPR. + """ + self._test_method = test_method + self.testcases = testcases + self._naming_type = naming_type + + def __call__(self, *args, **kwargs): + raise RuntimeError('You appear to be running a parameterized test case ' + 'without having inherited from parameterized.' + 'ParameterizedTestCase. This is bad because none of ' + 'your test cases are actually being run.') + + def __iter__(self): + test_method = self._test_method + naming_type = self._naming_type + + def MakeBoundParamTest(testcase_params): + @functools.wraps(test_method) + def BoundParamTest(self): + if isinstance(testcase_params, collections.Mapping): + test_method(self, **testcase_params) + elif _NonStringIterable(testcase_params): + test_method(self, *testcase_params) + else: + test_method(self, testcase_params) + + if naming_type is _FIRST_ARG: + # Signal the metaclass that the name of the test function is unique + # and descriptive. + BoundParamTest.__x_use_name__ = True + BoundParamTest.__name__ += str(testcase_params[0]) + testcase_params = testcase_params[1:] + elif naming_type is _ARGUMENT_REPR: + # __x_extra_id__ is used to pass naming information to the __new__ + # method of TestGeneratorMetaclass. + # The metaclass will make sure to create a unique, but nondescriptive + # name for this test. + BoundParamTest.__x_extra_id__ = '(%s)' % ( + _FormatParameterList(testcase_params),) + else: + raise RuntimeError('%s is not a valid naming type.' % (naming_type,)) + + BoundParamTest.__doc__ = '%s(%s)' % ( + BoundParamTest.__name__, _FormatParameterList(testcase_params)) + if test_method.__doc__: + BoundParamTest.__doc__ += '\n%s' % (test_method.__doc__,) + return BoundParamTest + return (MakeBoundParamTest(c) for c in self.testcases) + + +def _IsSingletonList(testcases): + """True iff testcases contains only a single non-tuple element.""" + return len(testcases) == 1 and not isinstance(testcases[0], tuple) + + +def _ModifyClass(class_object, testcases, naming_type): + assert not getattr(class_object, '_id_suffix', None), ( + 'Cannot add parameters to %s,' + ' which already has parameterized methods.' % (class_object,)) + class_object._id_suffix = id_suffix = {} + for name, obj in class_object.__dict__.items(): + if (name.startswith(unittest.TestLoader.testMethodPrefix) + and isinstance(obj, types.FunctionType)): + delattr(class_object, name) + methods = {} + _UpdateClassDictForParamTestCase( + methods, id_suffix, name, + _ParameterizedTestIter(obj, testcases, naming_type)) + for name, meth in methods.iteritems(): + setattr(class_object, name, meth) + + +def _ParameterDecorator(naming_type, testcases): + """Implementation of the parameterization decorators. + + Args: + naming_type: The naming type. + testcases: Testcase parameters. + + Returns: + A function for modifying the decorated object. + """ + def _Apply(obj): + if isinstance(obj, type): + _ModifyClass( + obj, + list(testcases) if not isinstance(testcases, collections.Sequence) + else testcases, + naming_type) + return obj + else: + return _ParameterizedTestIter(obj, testcases, naming_type) + + if _IsSingletonList(testcases): + assert _NonStringIterable(testcases[0]), ( + 'Single parameter argument must be a non-string iterable') + testcases = testcases[0] + + return _Apply + + +def Parameters(*testcases): + """A decorator for creating parameterized tests. + + See the module docstring for a usage example. + Args: + *testcases: Parameters for the decorated method, either a single + iterable, or a list of tuples/dicts/objects (for tests + with only one argument). + + Returns: + A test generator to be handled by TestGeneratorMetaclass. + """ + return _ParameterDecorator(_ARGUMENT_REPR, testcases) + + +def NamedParameters(*testcases): + """A decorator for creating parameterized tests. + + See the module docstring for a usage example. The first element of + each parameter tuple should be a string and will be appended to the + name of the test method. + + Args: + *testcases: Parameters for the decorated method, either a single + iterable, or a list of tuples. + + Returns: + A test generator to be handled by TestGeneratorMetaclass. + """ + return _ParameterDecorator(_FIRST_ARG, testcases) + + +class TestGeneratorMetaclass(type): + """Metaclass for test cases with test generators. + + A test generator is an iterable in a testcase that produces callables. These + callables must be single-argument methods. These methods are injected into + the class namespace and the original iterable is removed. If the name of the + iterable conforms to the test pattern, the injected methods will be picked + up as tests by the unittest framework. + + In general, it is supposed to be used in conjuction with the + Parameters decorator. + """ + + def __new__(mcs, class_name, bases, dct): + dct['_id_suffix'] = id_suffix = {} + for name, obj in dct.items(): + if (name.startswith(unittest.TestLoader.testMethodPrefix) and + _NonStringIterable(obj)): + iterator = iter(obj) + dct.pop(name) + _UpdateClassDictForParamTestCase(dct, id_suffix, name, iterator) + + return type.__new__(mcs, class_name, bases, dct) + + +def _UpdateClassDictForParamTestCase(dct, id_suffix, name, iterator): + """Adds individual test cases to a dictionary. + + Args: + dct: The target dictionary. + id_suffix: The dictionary for mapping names to test IDs. + name: The original name of the test case. + iterator: The iterator generating the individual test cases. + """ + for idx, func in enumerate(iterator): + assert callable(func), 'Test generators must yield callables, got %r' % ( + func,) + if getattr(func, '__x_use_name__', False): + new_name = func.__name__ + else: + new_name = '%s%s%d' % (name, _SEPARATOR, idx) + assert new_name not in dct, ( + 'Name of parameterized test case "%s" not unique' % (new_name,)) + dct[new_name] = func + id_suffix[new_name] = getattr(func, '__x_extra_id__', '') + + +class ParameterizedTestCase(basetest.TestCase): + """Base class for test cases using the Parameters decorator.""" + __metaclass__ = TestGeneratorMetaclass + + def _OriginalName(self): + return self._testMethodName.split(_SEPARATOR)[0] + + def __str__(self): + return '%s (%s)' % (self._OriginalName(), _StrClass(self.__class__)) + + def id(self): # pylint: disable=invalid-name + """Returns the descriptive ID of the test. + + This is used internally by the unittesting framework to get a name + for the test to be used in reports. + + Returns: + The test id. + """ + return '%s.%s%s' % (_StrClass(self.__class__), + self._OriginalName(), + self._id_suffix.get(self._testMethodName, '')) + + +def CoopParameterizedTestCase(other_base_class): + """Returns a new base class with a cooperative metaclass base. + + This enables the ParameterizedTestCase to be used in combination + with other base classes that have custom metaclasses, such as + mox.MoxTestBase. + + Only works with metaclasses that do not override type.__new__. + + Example: + + import google3 + import mox + + from google3.testing.pybase import parameterized + + class ExampleTest(parameterized.CoopParameterizedTestCase(mox.MoxTestBase)): + ... + + Args: + other_base_class: (class) A test case base class. + + Returns: + A new class object. + """ + metaclass = type( + 'CoopMetaclass', + (other_base_class.__metaclass__, + TestGeneratorMetaclass), {}) + return metaclass( + 'CoopParameterizedTestCase', + (other_base_class, ParameterizedTestCase), {}) diff --git a/python/google/protobuf/internal/api_implementation.cc b/python/google/protobuf/internal/api_implementation.cc index 83db40b1..6db12e8d 100644 --- a/python/google/protobuf/internal/api_implementation.cc +++ b/python/google/protobuf/internal/api_implementation.cc @@ -50,10 +50,7 @@ namespace python { // and // PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION=2 #ifdef PYTHON_PROTO2_CPP_IMPL_V1 -#if PY_MAJOR_VERSION >= 3 -#error "PYTHON_PROTO2_CPP_IMPL_V1 is not supported under Python 3." -#endif -static int kImplVersion = 1; +#error "PYTHON_PROTO2_CPP_IMPL_V1 is no longer supported." #else #ifdef PYTHON_PROTO2_CPP_IMPL_V2 static int kImplVersion = 2; @@ -62,14 +59,7 @@ static int kImplVersion = 2; static int kImplVersion = 0; #else -// The defaults are set here. Python 3 uses the fast C++ APIv2 by default. -// Python 2 still uses the Python version by default until some compatibility -// issues can be worked around. -#if PY_MAJOR_VERSION >= 3 -static int kImplVersion = 2; -#else -static int kImplVersion = 0; -#endif +static int kImplVersion = -1; // -1 means "Unspecified by compiler flags". #endif // PYTHON_PROTO2_PYTHON_IMPL #endif // PYTHON_PROTO2_CPP_IMPL_V2 diff --git a/python/google/protobuf/internal/api_implementation.py b/python/google/protobuf/internal/api_implementation.py index f7926c16..8ba4357c 100755 --- a/python/google/protobuf/internal/api_implementation.py +++ b/python/google/protobuf/internal/api_implementation.py @@ -40,14 +40,33 @@ try: # The compile-time constants in the _api_implementation module can be used to # switch to a certain implementation of the Python API at build time. _api_version = _api_implementation.api_version - del _api_implementation + _proto_extension_modules_exist_in_build = True except ImportError: - _api_version = 0 + _api_version = -1 # Unspecified by compiler flags. + _proto_extension_modules_exist_in_build = False + +if _api_version == 1: + raise ValueError('api_version=1 is no longer supported.') +if _api_version < 0: # Still unspecified? + try: + # The presence of this module in a build allows the proto implementation to + # be upgraded merely via build deps rather than a compiler flag or the + # runtime environment variable. + # pylint: disable=g-import-not-at-top + from google.protobuf import _use_fast_cpp_protos + # Work around a known issue in the classic bootstrap .par import hook. + if not _use_fast_cpp_protos: + raise ImportError('_use_fast_cpp_protos import succeeded but was None') + del _use_fast_cpp_protos + _api_version = 2 + except ImportError: + if _proto_extension_modules_exist_in_build: + if sys.version_info[0] >= 3: # Python 3 defaults to C++ impl v2. + _api_version = 2 + # TODO(b/17427486): Make Python 2 default to C++ impl v2. _default_implementation_type = ( - 'python' if _api_version == 0 else 'cpp') -_default_version_str = ( - '1' if _api_version <= 1 else '2') + 'python' if _api_version <= 0 else 'cpp') # This environment variable can be used to switch to a certain implementation # of the Python API, overriding the compile-time constants in the @@ -64,13 +83,12 @@ if _implementation_type != 'python': # _api_implementation module. Right now only 1 and 2 are valid values. Any other # value will be ignored. _implementation_version_str = os.getenv( - 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION', - _default_version_str) + 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION', '2') -if _implementation_version_str not in ('1', '2'): +if _implementation_version_str != '2': raise ValueError( - "unsupported PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION: '" + - _implementation_version_str + "' (supported versions: 1, 2)" + 'unsupported PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION: "' + + _implementation_version_str + '" (supported versions: 2)' ) _implementation_version = int(_implementation_version_str) diff --git a/python/google/protobuf/internal/api_implementation_default_test.py b/python/google/protobuf/internal/api_implementation_default_test.py deleted file mode 100644 index 78d5cf23..00000000 --- a/python/google/protobuf/internal/api_implementation_default_test.py +++ /dev/null @@ -1,63 +0,0 @@ -#! /usr/bin/python -# -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Test that the api_implementation defaults are what we expect.""" - -import os -import sys -# Clear environment implementation settings before the google3 imports. -os.environ.pop('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION', None) -os.environ.pop('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION', None) - -# pylint: disable=g-import-not-at-top -from google.apputils import basetest -from google.protobuf.internal import api_implementation - - -class ApiImplementationDefaultTest(basetest.TestCase): - - if sys.version_info.major <= 2: - - def testThatPythonIsTheDefault(self): - """If -DPYTHON_PROTO_*IMPL* was given at build time, this may fail.""" - self.assertEqual('python', api_implementation.Type()) - - else: - - def testThatCppApiV2IsTheDefault(self): - """If -DPYTHON_PROTO_*IMPL* was given at build time, this may fail.""" - self.assertEqual('cpp', api_implementation.Type()) - self.assertEqual(2, api_implementation.Version()) - - -if __name__ == '__main__': - basetest.main() diff --git a/python/google/protobuf/internal/containers.py b/python/google/protobuf/internal/containers.py index 20bfa857..d976f9e1 100755 --- a/python/google/protobuf/internal/containers.py +++ b/python/google/protobuf/internal/containers.py @@ -41,7 +41,6 @@ are: __author__ = 'petar@google.com (Petar Petrov)' - class BaseContainer(object): """Base container class.""" @@ -119,15 +118,23 @@ class RepeatedScalarFieldContainer(BaseContainer): self._message_listener.Modified() def extend(self, elem_seq): - """Extends by appending the given sequence. Similar to list.extend().""" - if not elem_seq: - return + """Extends by appending the given iterable. Similar to list.extend().""" - new_values = [] - for elem in elem_seq: - new_values.append(self._type_checker.CheckValue(elem)) - self._values.extend(new_values) - self._message_listener.Modified() + if elem_seq is None: + return + try: + elem_seq_iter = iter(elem_seq) + except TypeError: + if not elem_seq: + # silently ignore falsy inputs :-/. + # TODO(ptucker): Deprecate this behavior. b/18413862 + return + raise + + new_values = [self._type_checker.CheckValue(elem) for elem in elem_seq_iter] + if new_values: + self._values.extend(new_values) + self._message_listener.Modified() def MergeFrom(self, other): """Appends the contents of another repeated field of the same type to this @@ -141,6 +148,12 @@ class RepeatedScalarFieldContainer(BaseContainer): self._values.remove(elem) self._message_listener.Modified() + def pop(self, key=-1): + """Removes and returns an item at a given index. Similar to list.pop().""" + value = self._values[key] + self.__delitem__(key) + return value + def __setitem__(self, key, value): """Sets the item on the specified position.""" if isinstance(key, slice): # PY3 @@ -245,6 +258,12 @@ class RepeatedCompositeFieldContainer(BaseContainer): self._values.remove(elem) self._message_listener.Modified() + def pop(self, key=-1): + """Removes and returns an item at a given index. Similar to list.pop().""" + value = self._values[key] + self.__delitem__(key) + return value + def __getslice__(self, start, stop): """Retrieves the subset of items from between the specified indices.""" return self._values[start:stop] diff --git a/python/google/protobuf/internal/decoder.py b/python/google/protobuf/internal/decoder.py index a4b90608..0f500606 100755 --- a/python/google/protobuf/internal/decoder.py +++ b/python/google/protobuf/internal/decoder.py @@ -621,9 +621,6 @@ def MessageDecoder(field_number, is_repeated, is_packed, key, new_default): if value is None: value = field_dict.setdefault(key, new_default(message)) while 1: - value = field_dict.get(key) - if value is None: - value = field_dict.setdefault(key, new_default(message)) # Read length. (size, pos) = local_DecodeVarint(buffer, pos) new_pos = pos + size diff --git a/python/google/protobuf/internal/descriptor_test.py b/python/google/protobuf/internal/descriptor_test.py index 3924f21a..50c4dbba 100755 --- a/python/google/protobuf/internal/descriptor_test.py +++ b/python/google/protobuf/internal/descriptor_test.py @@ -34,12 +34,16 @@ __author__ = 'robinson@google.com (Will Robinson)' +import sys + from google.apputils import basetest from google.protobuf import unittest_custom_options_pb2 from google.protobuf import unittest_import_pb2 from google.protobuf import unittest_pb2 from google.protobuf import descriptor_pb2 +from google.protobuf.internal import api_implementation from google.protobuf import descriptor +from google.protobuf import symbol_database from google.protobuf import text_format @@ -51,41 +55,28 @@ name: 'TestEmptyMessage' class DescriptorTest(basetest.TestCase): def setUp(self): - self.my_file = descriptor.FileDescriptor( + file_proto = descriptor_pb2.FileDescriptorProto( name='some/filename/some.proto', - package='protobuf_unittest' - ) - self.my_enum = descriptor.EnumDescriptor( - name='ForeignEnum', - full_name='protobuf_unittest.ForeignEnum', - filename=None, - file=self.my_file, - values=[ - descriptor.EnumValueDescriptor(name='FOREIGN_FOO', index=0, number=4), - descriptor.EnumValueDescriptor(name='FOREIGN_BAR', index=1, number=5), - descriptor.EnumValueDescriptor(name='FOREIGN_BAZ', index=2, number=6), - ]) - self.my_message = descriptor.Descriptor( - name='NestedMessage', - full_name='protobuf_unittest.TestAllTypes.NestedMessage', - filename=None, - file=self.my_file, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='bb', - full_name='protobuf_unittest.TestAllTypes.NestedMessage.bb', - index=0, number=1, - type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None), - ], - nested_types=[], - enum_types=[ - self.my_enum, - ], - extensions=[]) + package='protobuf_unittest') + message_proto = file_proto.message_type.add( + name='NestedMessage') + message_proto.field.add( + name='bb', + number=1, + type=descriptor_pb2.FieldDescriptorProto.TYPE_INT32, + label=descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL) + enum_proto = message_proto.enum_type.add( + name='ForeignEnum') + enum_proto.value.add(name='FOREIGN_FOO', number=4) + enum_proto.value.add(name='FOREIGN_BAR', number=5) + enum_proto.value.add(name='FOREIGN_BAZ', number=6) + + descriptor_pool = symbol_database.Default().pool + descriptor_pool.Add(file_proto) + self.my_file = descriptor_pool.FindFileByName(file_proto.name) + self.my_message = self.my_file.message_types_by_name[message_proto.name] + self.my_enum = self.my_message.enum_types_by_name[enum_proto.name] + self.my_method = descriptor.MethodDescriptor( name='Bar', full_name='protobuf_unittest.TestService.Bar', @@ -173,6 +164,11 @@ class DescriptorTest(basetest.TestCase): self.assertEqual(unittest_custom_options_pb2.METHODOPT1_VAL2, method_options.Extensions[method_opt1]) + message_descriptor = ( + unittest_custom_options_pb2.DummyMessageContainingEnum.DESCRIPTOR) + self.assertTrue(file_descriptor.has_options) + self.assertFalse(message_descriptor.has_options) + def testDifferentCustomOptionTypes(self): kint32min = -2**31 kint64min = -2**63 @@ -394,6 +390,108 @@ class DescriptorTest(basetest.TestCase): self.assertEqual(self.my_file.name, 'some/filename/some.proto') self.assertEqual(self.my_file.package, 'protobuf_unittest') + @basetest.unittest.skipIf( + api_implementation.Type() != 'cpp' or api_implementation.Version() != 2, + 'Immutability of descriptors is only enforced in v2 implementation') + def testImmutableCppDescriptor(self): + message_descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR + with self.assertRaises(AttributeError): + message_descriptor.fields_by_name = None + with self.assertRaises(TypeError): + message_descriptor.fields_by_name['Another'] = None + with self.assertRaises(TypeError): + message_descriptor.fields.append(None) + + +class GeneratedDescriptorTest(basetest.TestCase): + """Tests for the properties of descriptors in generated code.""" + + def CheckMessageDescriptor(self, message_descriptor): + # Basic properties + self.assertEqual(message_descriptor.name, 'TestAllTypes') + self.assertEqual(message_descriptor.full_name, + 'protobuf_unittest.TestAllTypes') + # Test equality and hashability + self.assertEqual(message_descriptor, message_descriptor) + self.assertEqual(message_descriptor.fields[0].containing_type, + message_descriptor) + self.assertIn(message_descriptor, [message_descriptor]) + self.assertIn(message_descriptor, {message_descriptor: None}) + # Test field containers + self.CheckDescriptorSequence(message_descriptor.fields) + self.CheckDescriptorMapping(message_descriptor.fields_by_name) + self.CheckDescriptorMapping(message_descriptor.fields_by_number) + + def CheckFieldDescriptor(self, field_descriptor): + # Basic properties + self.assertEqual(field_descriptor.name, 'optional_int32') + self.assertEqual(field_descriptor.full_name, + 'protobuf_unittest.TestAllTypes.optional_int32') + self.assertEqual(field_descriptor.containing_type.name, 'TestAllTypes') + # Test equality and hashability + self.assertEqual(field_descriptor, field_descriptor) + self.assertEqual( + field_descriptor.containing_type.fields_by_name['optional_int32'], + field_descriptor) + self.assertIn(field_descriptor, [field_descriptor]) + self.assertIn(field_descriptor, {field_descriptor: None}) + + def CheckDescriptorSequence(self, sequence): + # Verifies that a property like 'messageDescriptor.fields' has all the + # properties of an immutable abc.Sequence. + self.assertGreater(len(sequence), 0) # Sized + self.assertEqual(len(sequence), len(list(sequence))) # Iterable + item = sequence[0] + self.assertEqual(item, sequence[0]) + self.assertIn(item, sequence) # Container + self.assertEqual(sequence.index(item), 0) + self.assertEqual(sequence.count(item), 1) + reversed_iterator = reversed(sequence) + self.assertEqual(list(reversed_iterator), list(sequence)[::-1]) + self.assertRaises(StopIteration, next, reversed_iterator) + + def CheckDescriptorMapping(self, mapping): + # Verifies that a property like 'messageDescriptor.fields' has all the + # properties of an immutable abc.Mapping. + self.assertGreater(len(mapping), 0) # Sized + self.assertEqual(len(mapping), len(list(mapping))) # Iterable + if sys.version_info.major >= 3: + key, item = next(iter(mapping.items())) + else: + key, item = mapping.items()[0] + self.assertIn(key, mapping) # Container + self.assertEqual(mapping.get(key), item) + # keys(), iterkeys() &co + item = (next(iter(mapping.keys())), next(iter(mapping.values()))) + self.assertEqual(item, next(iter(mapping.items()))) + if sys.version_info.major < 3: + def CheckItems(seq, iterator): + self.assertEqual(next(iterator), seq[0]) + self.assertEqual(list(iterator), seq[1:]) + CheckItems(mapping.keys(), mapping.iterkeys()) + CheckItems(mapping.values(), mapping.itervalues()) + CheckItems(mapping.items(), mapping.iteritems()) + + def testDescriptor(self): + message_descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR + self.CheckMessageDescriptor(message_descriptor) + field_descriptor = message_descriptor.fields_by_name['optional_int32'] + self.CheckFieldDescriptor(field_descriptor) + + def testCppDescriptorContainer(self): + # Check that the collection is still valid even if the parent disappeared. + enum = unittest_pb2.TestAllTypes.DESCRIPTOR.enum_types_by_name['NestedEnum'] + values = enum.values + del enum + self.assertEqual('FOO', values[0].name) + + def testCppDescriptorContainer_Iterator(self): + # Same test with the iterator + enum = unittest_pb2.TestAllTypes.DESCRIPTOR.enum_types_by_name['NestedEnum'] + values_iter = iter(enum.values) + del enum + self.assertEqual('FOO', next(values_iter).name) + class DescriptorCopyToProtoTest(basetest.TestCase): """Tests for CopyTo functions of Descriptor.""" @@ -588,10 +686,12 @@ class DescriptorCopyToProtoTest(basetest.TestCase): output_type: '.protobuf_unittest.BarResponse' > """ - self._InternalTestCopyToProto( - unittest_pb2.TestService.DESCRIPTOR, - descriptor_pb2.ServiceDescriptorProto, - TEST_SERVICE_ASCII) + # TODO(rocking): enable this test after the proto descriptor change is + # checked in. + #self._InternalTestCopyToProto( + # unittest_pb2.TestService.DESCRIPTOR, + # descriptor_pb2.ServiceDescriptorProto, + # TEST_SERVICE_ASCII) class MakeDescriptorTest(basetest.TestCase): diff --git a/python/google/protobuf/internal/message_test.py b/python/google/protobuf/internal/message_test.py index 42e2ad7e..4d4de510 100755 --- a/python/google/protobuf/internal/message_test.py +++ b/python/google/protobuf/internal/message_test.py @@ -48,10 +48,13 @@ import math import operator import pickle import sys +import unittest from google.apputils import basetest from google.protobuf import unittest_pb2 +from google.protobuf import unittest_proto3_arena_pb2 from google.protobuf.internal import api_implementation +from google.protobuf.internal import _parameterized from google.protobuf.internal import test_util from google.protobuf import message @@ -69,88 +72,65 @@ def IsNegInf(val): return isinf(val) and (val < 0) +@_parameterized.Parameters( + (unittest_pb2), + (unittest_proto3_arena_pb2)) class MessageTest(basetest.TestCase): - def testBadUtf8String(self): + def testBadUtf8String(self, message_module): if api_implementation.Type() != 'python': self.skipTest("Skipping testBadUtf8String, currently only the python " "api implementation raises UnicodeDecodeError when a " "string field contains bad utf-8.") bad_utf8_data = test_util.GoldenFileData('bad_utf8_string') with self.assertRaises(UnicodeDecodeError) as context: - unittest_pb2.TestAllTypes.FromString(bad_utf8_data) - self.assertIn('field: protobuf_unittest.TestAllTypes.optional_string', - str(context.exception)) - - def testGoldenMessage(self): - golden_data = test_util.GoldenFileData( - 'golden_message_oneof_implemented') - golden_message = unittest_pb2.TestAllTypes() - golden_message.ParseFromString(golden_data) - test_util.ExpectAllFieldsSet(self, golden_message) - self.assertEqual(golden_data, golden_message.SerializeToString()) - golden_copy = copy.deepcopy(golden_message) - self.assertEqual(golden_data, golden_copy.SerializeToString()) + message_module.TestAllTypes.FromString(bad_utf8_data) + self.assertIn('TestAllTypes.optional_string', str(context.exception)) + + def testGoldenMessage(self, message_module): + # Proto3 doesn't have the "default_foo" members or foreign enums, + # and doesn't preserve unknown fields, so for proto3 we use a golden + # message that doesn't have these fields set. + if message_module is unittest_pb2: + golden_data = test_util.GoldenFileData( + 'golden_message_oneof_implemented') + else: + golden_data = test_util.GoldenFileData('golden_message_proto3') - def testGoldenExtensions(self): - golden_data = test_util.GoldenFileData('golden_message') - golden_message = unittest_pb2.TestAllExtensions() + golden_message = message_module.TestAllTypes() golden_message.ParseFromString(golden_data) - all_set = unittest_pb2.TestAllExtensions() - test_util.SetAllExtensions(all_set) - self.assertEquals(all_set, golden_message) + if message_module is unittest_pb2: + test_util.ExpectAllFieldsSet(self, golden_message) self.assertEqual(golden_data, golden_message.SerializeToString()) golden_copy = copy.deepcopy(golden_message) self.assertEqual(golden_data, golden_copy.SerializeToString()) - def testGoldenPackedMessage(self): + def testGoldenPackedMessage(self, message_module): golden_data = test_util.GoldenFileData('golden_packed_fields_message') - golden_message = unittest_pb2.TestPackedTypes() + golden_message = message_module.TestPackedTypes() golden_message.ParseFromString(golden_data) - all_set = unittest_pb2.TestPackedTypes() + all_set = message_module.TestPackedTypes() test_util.SetAllPackedFields(all_set) - self.assertEquals(all_set, golden_message) + self.assertEqual(all_set, golden_message) self.assertEqual(golden_data, all_set.SerializeToString()) golden_copy = copy.deepcopy(golden_message) self.assertEqual(golden_data, golden_copy.SerializeToString()) - def testGoldenPackedExtensions(self): - golden_data = test_util.GoldenFileData('golden_packed_fields_message') - golden_message = unittest_pb2.TestPackedExtensions() - golden_message.ParseFromString(golden_data) - all_set = unittest_pb2.TestPackedExtensions() - test_util.SetAllPackedExtensions(all_set) - self.assertEquals(all_set, golden_message) - self.assertEqual(golden_data, all_set.SerializeToString()) - golden_copy = copy.deepcopy(golden_message) - self.assertEqual(golden_data, golden_copy.SerializeToString()) - - def testPickleSupport(self): + def testPickleSupport(self, message_module): golden_data = test_util.GoldenFileData('golden_message') - golden_message = unittest_pb2.TestAllTypes() + golden_message = message_module.TestAllTypes() golden_message.ParseFromString(golden_data) pickled_message = pickle.dumps(golden_message) unpickled_message = pickle.loads(pickled_message) - self.assertEquals(unpickled_message, golden_message) - - - def testPickleIncompleteProto(self): - golden_message = unittest_pb2.TestRequired(a=1) - pickled_message = pickle.dumps(golden_message) - - unpickled_message = pickle.loads(pickled_message) - self.assertEquals(unpickled_message, golden_message) - self.assertEquals(unpickled_message.a, 1) - # This is still an incomplete proto - so serializing should fail - self.assertRaises(message.EncodeError, unpickled_message.SerializeToString) + self.assertEqual(unpickled_message, golden_message) - def testPositiveInfinity(self): + def testPositiveInfinity(self, message_module): golden_data = (b'\x5D\x00\x00\x80\x7F' b'\x61\x00\x00\x00\x00\x00\x00\xF0\x7F' b'\xCD\x02\x00\x00\x80\x7F' b'\xD1\x02\x00\x00\x00\x00\x00\x00\xF0\x7F') - golden_message = unittest_pb2.TestAllTypes() + golden_message = message_module.TestAllTypes() golden_message.ParseFromString(golden_data) self.assertTrue(IsPosInf(golden_message.optional_float)) self.assertTrue(IsPosInf(golden_message.optional_double)) @@ -158,12 +138,12 @@ class MessageTest(basetest.TestCase): self.assertTrue(IsPosInf(golden_message.repeated_double[0])) self.assertEqual(golden_data, golden_message.SerializeToString()) - def testNegativeInfinity(self): + def testNegativeInfinity(self, message_module): golden_data = (b'\x5D\x00\x00\x80\xFF' b'\x61\x00\x00\x00\x00\x00\x00\xF0\xFF' b'\xCD\x02\x00\x00\x80\xFF' b'\xD1\x02\x00\x00\x00\x00\x00\x00\xF0\xFF') - golden_message = unittest_pb2.TestAllTypes() + golden_message = message_module.TestAllTypes() golden_message.ParseFromString(golden_data) self.assertTrue(IsNegInf(golden_message.optional_float)) self.assertTrue(IsNegInf(golden_message.optional_double)) @@ -171,12 +151,12 @@ class MessageTest(basetest.TestCase): self.assertTrue(IsNegInf(golden_message.repeated_double[0])) self.assertEqual(golden_data, golden_message.SerializeToString()) - def testNotANumber(self): + def testNotANumber(self, message_module): golden_data = (b'\x5D\x00\x00\xC0\x7F' b'\x61\x00\x00\x00\x00\x00\x00\xF8\x7F' b'\xCD\x02\x00\x00\xC0\x7F' b'\xD1\x02\x00\x00\x00\x00\x00\x00\xF8\x7F') - golden_message = unittest_pb2.TestAllTypes() + golden_message = message_module.TestAllTypes() golden_message.ParseFromString(golden_data) self.assertTrue(isnan(golden_message.optional_float)) self.assertTrue(isnan(golden_message.optional_double)) @@ -188,47 +168,47 @@ class MessageTest(basetest.TestCase): # verify the serialized string can be converted into a correctly # behaving protocol buffer. serialized = golden_message.SerializeToString() - message = unittest_pb2.TestAllTypes() + message = message_module.TestAllTypes() message.ParseFromString(serialized) self.assertTrue(isnan(message.optional_float)) self.assertTrue(isnan(message.optional_double)) self.assertTrue(isnan(message.repeated_float[0])) self.assertTrue(isnan(message.repeated_double[0])) - def testPositiveInfinityPacked(self): + def testPositiveInfinityPacked(self, message_module): golden_data = (b'\xA2\x06\x04\x00\x00\x80\x7F' b'\xAA\x06\x08\x00\x00\x00\x00\x00\x00\xF0\x7F') - golden_message = unittest_pb2.TestPackedTypes() + golden_message = message_module.TestPackedTypes() golden_message.ParseFromString(golden_data) self.assertTrue(IsPosInf(golden_message.packed_float[0])) self.assertTrue(IsPosInf(golden_message.packed_double[0])) self.assertEqual(golden_data, golden_message.SerializeToString()) - def testNegativeInfinityPacked(self): + def testNegativeInfinityPacked(self, message_module): golden_data = (b'\xA2\x06\x04\x00\x00\x80\xFF' b'\xAA\x06\x08\x00\x00\x00\x00\x00\x00\xF0\xFF') - golden_message = unittest_pb2.TestPackedTypes() + golden_message = message_module.TestPackedTypes() golden_message.ParseFromString(golden_data) self.assertTrue(IsNegInf(golden_message.packed_float[0])) self.assertTrue(IsNegInf(golden_message.packed_double[0])) self.assertEqual(golden_data, golden_message.SerializeToString()) - def testNotANumberPacked(self): + def testNotANumberPacked(self, message_module): golden_data = (b'\xA2\x06\x04\x00\x00\xC0\x7F' b'\xAA\x06\x08\x00\x00\x00\x00\x00\x00\xF8\x7F') - golden_message = unittest_pb2.TestPackedTypes() + golden_message = message_module.TestPackedTypes() golden_message.ParseFromString(golden_data) self.assertTrue(isnan(golden_message.packed_float[0])) self.assertTrue(isnan(golden_message.packed_double[0])) serialized = golden_message.SerializeToString() - message = unittest_pb2.TestPackedTypes() + message = message_module.TestPackedTypes() message.ParseFromString(serialized) self.assertTrue(isnan(message.packed_float[0])) self.assertTrue(isnan(message.packed_double[0])) - def testExtremeFloatValues(self): - message = unittest_pb2.TestAllTypes() + def testExtremeFloatValues(self, message_module): + message = message_module.TestAllTypes() # Most positive exponent, no significand bits set. kMostPosExponentNoSigBits = math.pow(2, 127) @@ -272,8 +252,8 @@ class MessageTest(basetest.TestCase): message.ParseFromString(message.SerializeToString()) self.assertTrue(message.optional_float == -kMostNegExponentOneSigBit) - def testExtremeDoubleValues(self): - message = unittest_pb2.TestAllTypes() + def testExtremeDoubleValues(self, message_module): + message = message_module.TestAllTypes() # Most positive exponent, no significand bits set. kMostPosExponentNoSigBits = math.pow(2, 1023) @@ -317,43 +297,43 @@ class MessageTest(basetest.TestCase): message.ParseFromString(message.SerializeToString()) self.assertTrue(message.optional_double == -kMostNegExponentOneSigBit) - def testFloatPrinting(self): - message = unittest_pb2.TestAllTypes() + def testFloatPrinting(self, message_module): + message = message_module.TestAllTypes() message.optional_float = 2.0 self.assertEqual(str(message), 'optional_float: 2.0\n') - def testHighPrecisionFloatPrinting(self): - message = unittest_pb2.TestAllTypes() + def testHighPrecisionFloatPrinting(self, message_module): + message = message_module.TestAllTypes() message.optional_double = 0.12345678912345678 if sys.version_info.major >= 3: self.assertEqual(str(message), 'optional_double: 0.12345678912345678\n') else: self.assertEqual(str(message), 'optional_double: 0.123456789123\n') - def testUnknownFieldPrinting(self): - populated = unittest_pb2.TestAllTypes() + def testUnknownFieldPrinting(self, message_module): + populated = message_module.TestAllTypes() test_util.SetAllNonLazyFields(populated) - empty = unittest_pb2.TestEmptyMessage() + empty = message_module.TestEmptyMessage() empty.ParseFromString(populated.SerializeToString()) self.assertEqual(str(empty), '') - def testRepeatedNestedFieldIteration(self): - msg = unittest_pb2.TestAllTypes() + def testRepeatedNestedFieldIteration(self, message_module): + msg = message_module.TestAllTypes() msg.repeated_nested_message.add(bb=1) msg.repeated_nested_message.add(bb=2) msg.repeated_nested_message.add(bb=3) msg.repeated_nested_message.add(bb=4) - self.assertEquals([1, 2, 3, 4], - [m.bb for m in msg.repeated_nested_message]) - self.assertEquals([4, 3, 2, 1], - [m.bb for m in reversed(msg.repeated_nested_message)]) - self.assertEquals([4, 3, 2, 1], - [m.bb for m in msg.repeated_nested_message[::-1]]) + self.assertEqual([1, 2, 3, 4], + [m.bb for m in msg.repeated_nested_message]) + self.assertEqual([4, 3, 2, 1], + [m.bb for m in reversed(msg.repeated_nested_message)]) + self.assertEqual([4, 3, 2, 1], + [m.bb for m in msg.repeated_nested_message[::-1]]) - def testSortingRepeatedScalarFieldsDefaultComparator(self): + def testSortingRepeatedScalarFieldsDefaultComparator(self, message_module): """Check some different types with the default comparator.""" - message = unittest_pb2.TestAllTypes() + message = message_module.TestAllTypes() # TODO(mattp): would testing more scalar types strengthen test? message.repeated_int32.append(1) @@ -388,9 +368,9 @@ class MessageTest(basetest.TestCase): self.assertEqual(message.repeated_bytes[1], b'b') self.assertEqual(message.repeated_bytes[2], b'c') - def testSortingRepeatedScalarFieldsCustomComparator(self): + def testSortingRepeatedScalarFieldsCustomComparator(self, message_module): """Check some different types with custom comparator.""" - message = unittest_pb2.TestAllTypes() + message = message_module.TestAllTypes() message.repeated_int32.append(-3) message.repeated_int32.append(-2) @@ -408,9 +388,9 @@ class MessageTest(basetest.TestCase): self.assertEqual(message.repeated_string[1], 'bb') self.assertEqual(message.repeated_string[2], 'aaa') - def testSortingRepeatedCompositeFieldsCustomComparator(self): + def testSortingRepeatedCompositeFieldsCustomComparator(self, message_module): """Check passing a custom comparator to sort a repeated composite field.""" - message = unittest_pb2.TestAllTypes() + message = message_module.TestAllTypes() message.repeated_nested_message.add().bb = 1 message.repeated_nested_message.add().bb = 3 @@ -426,9 +406,9 @@ class MessageTest(basetest.TestCase): self.assertEqual(message.repeated_nested_message[4].bb, 5) self.assertEqual(message.repeated_nested_message[5].bb, 6) - def testRepeatedCompositeFieldSortArguments(self): + def testRepeatedCompositeFieldSortArguments(self, message_module): """Check sorting a repeated composite field using list.sort() arguments.""" - message = unittest_pb2.TestAllTypes() + message = message_module.TestAllTypes() get_bb = operator.attrgetter('bb') cmp_bb = lambda a, b: cmp(a.bb, b.bb) @@ -452,9 +432,9 @@ class MessageTest(basetest.TestCase): self.assertEqual([k.bb for k in message.repeated_nested_message], [6, 5, 4, 3, 2, 1]) - def testRepeatedScalarFieldSortArguments(self): + def testRepeatedScalarFieldSortArguments(self, message_module): """Check sorting a scalar field using list.sort() arguments.""" - message = unittest_pb2.TestAllTypes() + message = message_module.TestAllTypes() message.repeated_int32.append(-3) message.repeated_int32.append(-2) @@ -484,9 +464,9 @@ class MessageTest(basetest.TestCase): message.repeated_string.sort(cmp=len_cmp, reverse=True) self.assertEqual(list(message.repeated_string), ['aaa', 'bb', 'c']) - def testRepeatedFieldsComparable(self): - m1 = unittest_pb2.TestAllTypes() - m2 = unittest_pb2.TestAllTypes() + def testRepeatedFieldsComparable(self, message_module): + m1 = message_module.TestAllTypes() + m2 = message_module.TestAllTypes() m1.repeated_int32.append(0) m1.repeated_int32.append(1) m1.repeated_int32.append(2) @@ -519,55 +499,6 @@ class MessageTest(basetest.TestCase): # TODO(anuraag): Implement extensiondict comparison in C++ and then add test - def testParsingMerge(self): - """Check the merge behavior when a required or optional field appears - multiple times in the input.""" - messages = [ - unittest_pb2.TestAllTypes(), - unittest_pb2.TestAllTypes(), - unittest_pb2.TestAllTypes() ] - messages[0].optional_int32 = 1 - messages[1].optional_int64 = 2 - messages[2].optional_int32 = 3 - messages[2].optional_string = 'hello' - - merged_message = unittest_pb2.TestAllTypes() - merged_message.optional_int32 = 3 - merged_message.optional_int64 = 2 - merged_message.optional_string = 'hello' - - generator = unittest_pb2.TestParsingMerge.RepeatedFieldsGenerator() - generator.field1.extend(messages) - generator.field2.extend(messages) - generator.field3.extend(messages) - generator.ext1.extend(messages) - generator.ext2.extend(messages) - generator.group1.add().field1.MergeFrom(messages[0]) - generator.group1.add().field1.MergeFrom(messages[1]) - generator.group1.add().field1.MergeFrom(messages[2]) - generator.group2.add().field1.MergeFrom(messages[0]) - generator.group2.add().field1.MergeFrom(messages[1]) - generator.group2.add().field1.MergeFrom(messages[2]) - - data = generator.SerializeToString() - parsing_merge = unittest_pb2.TestParsingMerge() - parsing_merge.ParseFromString(data) - - # Required and optional fields should be merged. - self.assertEqual(parsing_merge.required_all_types, merged_message) - self.assertEqual(parsing_merge.optional_all_types, merged_message) - self.assertEqual(parsing_merge.optionalgroup.optional_group_all_types, - merged_message) - self.assertEqual(parsing_merge.Extensions[ - unittest_pb2.TestParsingMerge.optional_ext], - merged_message) - - # Repeated fields should not be merged. - self.assertEqual(len(parsing_merge.repeated_all_types), 3) - self.assertEqual(len(parsing_merge.repeatedgroup), 3) - self.assertEqual(len(parsing_merge.Extensions[ - unittest_pb2.TestParsingMerge.repeated_ext]), 3) - def ensureNestedMessageExists(self, msg, attribute): """Make sure that a nested message object exists. @@ -577,12 +508,28 @@ class MessageTest(basetest.TestCase): getattr(msg, attribute) self.assertFalse(msg.HasField(attribute)) - def testOneofGetCaseNonexistingField(self): - m = unittest_pb2.TestAllTypes() + def testOneofGetCaseNonexistingField(self, message_module): + m = message_module.TestAllTypes() self.assertRaises(ValueError, m.WhichOneof, 'no_such_oneof_field') - def testOneofSemantics(self): - m = unittest_pb2.TestAllTypes() + def testOneofDefaultValues(self, message_module): + m = message_module.TestAllTypes() + self.assertIs(None, m.WhichOneof('oneof_field')) + self.assertFalse(m.HasField('oneof_uint32')) + + # Oneof is set even when setting it to a default value. + m.oneof_uint32 = 0 + self.assertEqual('oneof_uint32', m.WhichOneof('oneof_field')) + self.assertTrue(m.HasField('oneof_uint32')) + self.assertFalse(m.HasField('oneof_string')) + + m.oneof_string = "" + self.assertEqual('oneof_string', m.WhichOneof('oneof_field')) + self.assertTrue(m.HasField('oneof_string')) + self.assertFalse(m.HasField('oneof_uint32')) + + def testOneofSemantics(self, message_module): + m = message_module.TestAllTypes() self.assertIs(None, m.WhichOneof('oneof_field')) m.oneof_uint32 = 11 @@ -604,96 +551,568 @@ class MessageTest(basetest.TestCase): self.assertFalse(m.HasField('oneof_nested_message')) self.assertTrue(m.HasField('oneof_bytes')) - def testOneofCompositeFieldReadAccess(self): - m = unittest_pb2.TestAllTypes() + def testOneofCompositeFieldReadAccess(self, message_module): + m = message_module.TestAllTypes() m.oneof_uint32 = 11 self.ensureNestedMessageExists(m, 'oneof_nested_message') self.assertEqual('oneof_uint32', m.WhichOneof('oneof_field')) self.assertEqual(11, m.oneof_uint32) - def testOneofHasField(self): - m = unittest_pb2.TestAllTypes() - self.assertFalse(m.HasField('oneof_field')) + def testOneofWhichOneof(self, message_module): + m = message_module.TestAllTypes() + self.assertIs(None, m.WhichOneof('oneof_field')) + if message_module is unittest_pb2: + self.assertFalse(m.HasField('oneof_field')) + m.oneof_uint32 = 11 - self.assertTrue(m.HasField('oneof_field')) + self.assertEqual('oneof_uint32', m.WhichOneof('oneof_field')) + if message_module is unittest_pb2: + self.assertTrue(m.HasField('oneof_field')) + m.oneof_bytes = b'bb' - self.assertTrue(m.HasField('oneof_field')) + self.assertEqual('oneof_bytes', m.WhichOneof('oneof_field')) + m.ClearField('oneof_bytes') - self.assertFalse(m.HasField('oneof_field')) + self.assertIs(None, m.WhichOneof('oneof_field')) + if message_module is unittest_pb2: + self.assertFalse(m.HasField('oneof_field')) - def testOneofClearField(self): - m = unittest_pb2.TestAllTypes() + def testOneofClearField(self, message_module): + m = message_module.TestAllTypes() m.oneof_uint32 = 11 m.ClearField('oneof_field') - self.assertFalse(m.HasField('oneof_field')) + if message_module is unittest_pb2: + self.assertFalse(m.HasField('oneof_field')) self.assertFalse(m.HasField('oneof_uint32')) self.assertIs(None, m.WhichOneof('oneof_field')) - def testOneofClearSetField(self): - m = unittest_pb2.TestAllTypes() + def testOneofClearSetField(self, message_module): + m = message_module.TestAllTypes() m.oneof_uint32 = 11 m.ClearField('oneof_uint32') - self.assertFalse(m.HasField('oneof_field')) + if message_module is unittest_pb2: + self.assertFalse(m.HasField('oneof_field')) self.assertFalse(m.HasField('oneof_uint32')) self.assertIs(None, m.WhichOneof('oneof_field')) - def testOneofClearUnsetField(self): - m = unittest_pb2.TestAllTypes() + def testOneofClearUnsetField(self, message_module): + m = message_module.TestAllTypes() m.oneof_uint32 = 11 self.ensureNestedMessageExists(m, 'oneof_nested_message') m.ClearField('oneof_nested_message') self.assertEqual(11, m.oneof_uint32) - self.assertTrue(m.HasField('oneof_field')) + if message_module is unittest_pb2: + self.assertTrue(m.HasField('oneof_field')) self.assertTrue(m.HasField('oneof_uint32')) self.assertEqual('oneof_uint32', m.WhichOneof('oneof_field')) - def testOneofDeserialize(self): - m = unittest_pb2.TestAllTypes() + def testOneofDeserialize(self, message_module): + m = message_module.TestAllTypes() m.oneof_uint32 = 11 - m2 = unittest_pb2.TestAllTypes() + m2 = message_module.TestAllTypes() m2.ParseFromString(m.SerializeToString()) self.assertEqual('oneof_uint32', m2.WhichOneof('oneof_field')) - def testOneofCopyFrom(self): - m = unittest_pb2.TestAllTypes() + def testOneofCopyFrom(self, message_module): + m = message_module.TestAllTypes() m.oneof_uint32 = 11 - m2 = unittest_pb2.TestAllTypes() + m2 = message_module.TestAllTypes() m2.CopyFrom(m) self.assertEqual('oneof_uint32', m2.WhichOneof('oneof_field')) - def testOneofNestedMergeFrom(self): - m = unittest_pb2.NestedTestAllTypes() + def testOneofNestedMergeFrom(self, message_module): + m = message_module.NestedTestAllTypes() m.payload.oneof_uint32 = 11 - m2 = unittest_pb2.NestedTestAllTypes() + m2 = message_module.NestedTestAllTypes() m2.payload.oneof_bytes = b'bb' m2.child.payload.oneof_bytes = b'bb' m2.MergeFrom(m) self.assertEqual('oneof_uint32', m2.payload.WhichOneof('oneof_field')) self.assertEqual('oneof_bytes', m2.child.payload.WhichOneof('oneof_field')) - def testOneofClear(self): - m = unittest_pb2.TestAllTypes() + def testOneofMessageMergeFrom(self, message_module): + m = message_module.NestedTestAllTypes() + m.payload.oneof_nested_message.bb = 11 + m.child.payload.oneof_nested_message.bb = 12 + m2 = message_module.NestedTestAllTypes() + m2.payload.oneof_uint32 = 13 + m2.MergeFrom(m) + self.assertEqual('oneof_nested_message', + m2.payload.WhichOneof('oneof_field')) + self.assertEqual('oneof_nested_message', + m2.child.payload.WhichOneof('oneof_field')) + + def testOneofNestedMessageInit(self, message_module): + m = message_module.TestAllTypes( + oneof_nested_message=message_module.TestAllTypes.NestedMessage()) + self.assertEqual('oneof_nested_message', m.WhichOneof('oneof_field')) + + def testOneofClear(self, message_module): + m = message_module.TestAllTypes() m.oneof_uint32 = 11 m.Clear() self.assertIsNone(m.WhichOneof('oneof_field')) m.oneof_bytes = b'bb' - self.assertTrue(m.HasField('oneof_field')) + self.assertEqual('oneof_bytes', m.WhichOneof('oneof_field')) + + def testAssignByteStringToUnicodeField(self, message_module): + """Assigning a byte string to a string field should result + in the value being converted to a Unicode string.""" + m = message_module.TestAllTypes() + m.optional_string = str('') + self.assertTrue(isinstance(m.optional_string, unicode)) +# TODO(haberman): why are these tests Google-internal only? - def testSortEmptyRepeatedCompositeContainer(self): + def testLongValuedSlice(self, message_module): + """It should be possible to use long-valued indicies in slices + + This didn't used to work in the v2 C++ implementation. + """ + m = message_module.TestAllTypes() + + # Repeated scalar + m.repeated_int32.append(1) + sl = m.repeated_int32[long(0):long(len(m.repeated_int32))] + self.assertEqual(len(m.repeated_int32), len(sl)) + + # Repeated composite + m.repeated_nested_message.add().bb = 3 + sl = m.repeated_nested_message[long(0):long(len(m.repeated_nested_message))] + self.assertEqual(len(m.repeated_nested_message), len(sl)) + + def testExtendShouldNotSwallowExceptions(self, message_module): + """This didn't use to work in the v2 C++ implementation.""" + m = message_module.TestAllTypes() + with self.assertRaises(NameError) as _: + m.repeated_int32.extend(a for i in range(10)) # pylint: disable=undefined-variable + with self.assertRaises(NameError) as _: + m.repeated_nested_enum.extend( + a for i in range(10)) # pylint: disable=undefined-variable + + FALSY_VALUES = [None, False, 0, 0.0, b'', u'', bytearray(), [], {}, set()] + + def testExtendInt32WithNothing(self, message_module): + """Test no-ops extending repeated int32 fields.""" + m = message_module.TestAllTypes() + self.assertSequenceEqual([], m.repeated_int32) + + # TODO(ptucker): Deprecate this behavior. b/18413862 + for falsy_value in MessageTest.FALSY_VALUES: + m.repeated_int32.extend(falsy_value) + self.assertSequenceEqual([], m.repeated_int32) + + m.repeated_int32.extend([]) + self.assertSequenceEqual([], m.repeated_int32) + + def testExtendFloatWithNothing(self, message_module): + """Test no-ops extending repeated float fields.""" + m = message_module.TestAllTypes() + self.assertSequenceEqual([], m.repeated_float) + + # TODO(ptucker): Deprecate this behavior. b/18413862 + for falsy_value in MessageTest.FALSY_VALUES: + m.repeated_float.extend(falsy_value) + self.assertSequenceEqual([], m.repeated_float) + + m.repeated_float.extend([]) + self.assertSequenceEqual([], m.repeated_float) + + def testExtendStringWithNothing(self, message_module): + """Test no-ops extending repeated string fields.""" + m = message_module.TestAllTypes() + self.assertSequenceEqual([], m.repeated_string) + + # TODO(ptucker): Deprecate this behavior. b/18413862 + for falsy_value in MessageTest.FALSY_VALUES: + m.repeated_string.extend(falsy_value) + self.assertSequenceEqual([], m.repeated_string) + + m.repeated_string.extend([]) + self.assertSequenceEqual([], m.repeated_string) + + def testExtendInt32WithPythonList(self, message_module): + """Test extending repeated int32 fields with python lists.""" + m = message_module.TestAllTypes() + self.assertSequenceEqual([], m.repeated_int32) + m.repeated_int32.extend([0]) + self.assertSequenceEqual([0], m.repeated_int32) + m.repeated_int32.extend([1, 2]) + self.assertSequenceEqual([0, 1, 2], m.repeated_int32) + m.repeated_int32.extend([3, 4]) + self.assertSequenceEqual([0, 1, 2, 3, 4], m.repeated_int32) + + def testExtendFloatWithPythonList(self, message_module): + """Test extending repeated float fields with python lists.""" + m = message_module.TestAllTypes() + self.assertSequenceEqual([], m.repeated_float) + m.repeated_float.extend([0.0]) + self.assertSequenceEqual([0.0], m.repeated_float) + m.repeated_float.extend([1.0, 2.0]) + self.assertSequenceEqual([0.0, 1.0, 2.0], m.repeated_float) + m.repeated_float.extend([3.0, 4.0]) + self.assertSequenceEqual([0.0, 1.0, 2.0, 3.0, 4.0], m.repeated_float) + + def testExtendStringWithPythonList(self, message_module): + """Test extending repeated string fields with python lists.""" + m = message_module.TestAllTypes() + self.assertSequenceEqual([], m.repeated_string) + m.repeated_string.extend(['']) + self.assertSequenceEqual([''], m.repeated_string) + m.repeated_string.extend(['11', '22']) + self.assertSequenceEqual(['', '11', '22'], m.repeated_string) + m.repeated_string.extend(['33', '44']) + self.assertSequenceEqual(['', '11', '22', '33', '44'], m.repeated_string) + + def testExtendStringWithString(self, message_module): + """Test extending repeated string fields with characters from a string.""" + m = message_module.TestAllTypes() + self.assertSequenceEqual([], m.repeated_string) + m.repeated_string.extend('abc') + self.assertSequenceEqual(['a', 'b', 'c'], m.repeated_string) + + class TestIterable(object): + """This iterable object mimics the behavior of numpy.array. + + __nonzero__ fails for length > 1, and returns bool(item[0]) for length == 1. + + """ + + def __init__(self, values=None): + self._list = values or [] + + def __nonzero__(self): + size = len(self._list) + if size == 0: + return False + if size == 1: + return bool(self._list[0]) + raise ValueError('Truth value is ambiguous.') + + def __len__(self): + return len(self._list) + + def __iter__(self): + return self._list.__iter__() + + def testExtendInt32WithIterable(self, message_module): + """Test extending repeated int32 fields with iterable.""" + m = message_module.TestAllTypes() + self.assertSequenceEqual([], m.repeated_int32) + m.repeated_int32.extend(MessageTest.TestIterable([])) + self.assertSequenceEqual([], m.repeated_int32) + m.repeated_int32.extend(MessageTest.TestIterable([0])) + self.assertSequenceEqual([0], m.repeated_int32) + m.repeated_int32.extend(MessageTest.TestIterable([1, 2])) + self.assertSequenceEqual([0, 1, 2], m.repeated_int32) + m.repeated_int32.extend(MessageTest.TestIterable([3, 4])) + self.assertSequenceEqual([0, 1, 2, 3, 4], m.repeated_int32) + + def testExtendFloatWithIterable(self, message_module): + """Test extending repeated float fields with iterable.""" + m = message_module.TestAllTypes() + self.assertSequenceEqual([], m.repeated_float) + m.repeated_float.extend(MessageTest.TestIterable([])) + self.assertSequenceEqual([], m.repeated_float) + m.repeated_float.extend(MessageTest.TestIterable([0.0])) + self.assertSequenceEqual([0.0], m.repeated_float) + m.repeated_float.extend(MessageTest.TestIterable([1.0, 2.0])) + self.assertSequenceEqual([0.0, 1.0, 2.0], m.repeated_float) + m.repeated_float.extend(MessageTest.TestIterable([3.0, 4.0])) + self.assertSequenceEqual([0.0, 1.0, 2.0, 3.0, 4.0], m.repeated_float) + + def testExtendStringWithIterable(self, message_module): + """Test extending repeated string fields with iterable.""" + m = message_module.TestAllTypes() + self.assertSequenceEqual([], m.repeated_string) + m.repeated_string.extend(MessageTest.TestIterable([])) + self.assertSequenceEqual([], m.repeated_string) + m.repeated_string.extend(MessageTest.TestIterable([''])) + self.assertSequenceEqual([''], m.repeated_string) + m.repeated_string.extend(MessageTest.TestIterable(['1', '2'])) + self.assertSequenceEqual(['', '1', '2'], m.repeated_string) + m.repeated_string.extend(MessageTest.TestIterable(['3', '4'])) + self.assertSequenceEqual(['', '1', '2', '3', '4'], m.repeated_string) + + def testPickleRepeatedScalarContainer(self, message_module): + # TODO(tibell): The pure-Python implementation support pickling of + # scalar containers in *some* cases. For now the cpp2 version + # throws an exception to avoid a segfault. Investigate if we + # want to support pickling of these fields. + # + # For more information see: https://b2.corp.google.com/u/0/issues/18677897 + if (api_implementation.Type() != 'cpp' or + api_implementation.Version() == 2): + return + m = message_module.TestAllTypes() + with self.assertRaises(pickle.PickleError) as _: + pickle.dumps(m.repeated_int32, pickle.HIGHEST_PROTOCOL) + + + def testSortEmptyRepeatedCompositeContainer(self, message_module): """Exercise a scenario that has led to segfaults in the past. """ - m = unittest_pb2.TestAllTypes() + m = message_module.TestAllTypes() m.repeated_nested_message.sort() - def testHasFieldOnRepeatedField(self): + def testHasFieldOnRepeatedField(self, message_module): """Using HasField on a repeated field should raise an exception. """ - m = unittest_pb2.TestAllTypes() + m = message_module.TestAllTypes() with self.assertRaises(ValueError) as _: m.HasField('repeated_int32') + def testRepeatedScalarFieldPop(self, message_module): + m = message_module.TestAllTypes() + with self.assertRaises(IndexError) as _: + m.repeated_int32.pop() + m.repeated_int32.extend(range(5)) + self.assertEqual(4, m.repeated_int32.pop()) + self.assertEqual(0, m.repeated_int32.pop(0)) + self.assertEqual(2, m.repeated_int32.pop(1)) + self.assertEqual([1, 3], m.repeated_int32) + + def testRepeatedCompositeFieldPop(self, message_module): + m = message_module.TestAllTypes() + with self.assertRaises(IndexError) as _: + m.repeated_nested_message.pop() + for i in range(5): + n = m.repeated_nested_message.add() + n.bb = i + self.assertEqual(4, m.repeated_nested_message.pop().bb) + self.assertEqual(0, m.repeated_nested_message.pop(0).bb) + self.assertEqual(2, m.repeated_nested_message.pop(1).bb) + self.assertEqual([1, 3], [n.bb for n in m.repeated_nested_message]) + + +# Class to test proto2-only features (required, extensions, etc.) +class Proto2Test(basetest.TestCase): + + def testFieldPresence(self): + message = unittest_pb2.TestAllTypes() + + self.assertFalse(message.HasField("optional_int32")) + self.assertFalse(message.HasField("optional_bool")) + self.assertFalse(message.HasField("optional_nested_message")) + + with self.assertRaises(ValueError): + message.HasField("field_doesnt_exist") + + with self.assertRaises(ValueError): + message.HasField("repeated_int32") + with self.assertRaises(ValueError): + message.HasField("repeated_nested_message") + + self.assertEqual(0, message.optional_int32) + self.assertEqual(False, message.optional_bool) + self.assertEqual(0, message.optional_nested_message.bb) + + # Fields are set even when setting the values to default values. + message.optional_int32 = 0 + message.optional_bool = False + message.optional_nested_message.bb = 0 + self.assertTrue(message.HasField("optional_int32")) + self.assertTrue(message.HasField("optional_bool")) + self.assertTrue(message.HasField("optional_nested_message")) + + # Set the fields to non-default values. + message.optional_int32 = 5 + message.optional_bool = True + message.optional_nested_message.bb = 15 + + self.assertTrue(message.HasField("optional_int32")) + self.assertTrue(message.HasField("optional_bool")) + self.assertTrue(message.HasField("optional_nested_message")) + + # Clearing the fields unsets them and resets their value to default. + message.ClearField("optional_int32") + message.ClearField("optional_bool") + message.ClearField("optional_nested_message") + + self.assertFalse(message.HasField("optional_int32")) + self.assertFalse(message.HasField("optional_bool")) + self.assertFalse(message.HasField("optional_nested_message")) + self.assertEqual(0, message.optional_int32) + self.assertEqual(False, message.optional_bool) + self.assertEqual(0, message.optional_nested_message.bb) + + # TODO(tibell): The C++ implementations actually allows assignment + # of unknown enum values to *scalar* fields (but not repeated + # fields). Once checked enum fields becomes the default in the + # Python implementation, the C++ implementation should follow suit. + def testAssignInvalidEnum(self): + """It should not be possible to assign an invalid enum number to an + enum field.""" + m = unittest_pb2.TestAllTypes() + + with self.assertRaises(ValueError) as _: + m.optional_nested_enum = 1234567 + self.assertRaises(ValueError, m.repeated_nested_enum.append, 1234567) + + def testGoldenExtensions(self): + golden_data = test_util.GoldenFileData('golden_message') + golden_message = unittest_pb2.TestAllExtensions() + golden_message.ParseFromString(golden_data) + all_set = unittest_pb2.TestAllExtensions() + test_util.SetAllExtensions(all_set) + self.assertEqual(all_set, golden_message) + self.assertEqual(golden_data, golden_message.SerializeToString()) + golden_copy = copy.deepcopy(golden_message) + self.assertEqual(golden_data, golden_copy.SerializeToString()) + + def testGoldenPackedExtensions(self): + golden_data = test_util.GoldenFileData('golden_packed_fields_message') + golden_message = unittest_pb2.TestPackedExtensions() + golden_message.ParseFromString(golden_data) + all_set = unittest_pb2.TestPackedExtensions() + test_util.SetAllPackedExtensions(all_set) + self.assertEqual(all_set, golden_message) + self.assertEqual(golden_data, all_set.SerializeToString()) + golden_copy = copy.deepcopy(golden_message) + self.assertEqual(golden_data, golden_copy.SerializeToString()) + + def testPickleIncompleteProto(self): + golden_message = unittest_pb2.TestRequired(a=1) + pickled_message = pickle.dumps(golden_message) + + unpickled_message = pickle.loads(pickled_message) + self.assertEqual(unpickled_message, golden_message) + self.assertEqual(unpickled_message.a, 1) + # This is still an incomplete proto - so serializing should fail + self.assertRaises(message.EncodeError, unpickled_message.SerializeToString) + + # TODO(haberman): this isn't really a proto2-specific test except that this + # message has a required field in it. Should probably be factored out so + # that we can test the other parts with proto3. + def testParsingMerge(self): + """Check the merge behavior when a required or optional field appears + multiple times in the input.""" + messages = [ + unittest_pb2.TestAllTypes(), + unittest_pb2.TestAllTypes(), + unittest_pb2.TestAllTypes() ] + messages[0].optional_int32 = 1 + messages[1].optional_int64 = 2 + messages[2].optional_int32 = 3 + messages[2].optional_string = 'hello' + + merged_message = unittest_pb2.TestAllTypes() + merged_message.optional_int32 = 3 + merged_message.optional_int64 = 2 + merged_message.optional_string = 'hello' + + generator = unittest_pb2.TestParsingMerge.RepeatedFieldsGenerator() + generator.field1.extend(messages) + generator.field2.extend(messages) + generator.field3.extend(messages) + generator.ext1.extend(messages) + generator.ext2.extend(messages) + generator.group1.add().field1.MergeFrom(messages[0]) + generator.group1.add().field1.MergeFrom(messages[1]) + generator.group1.add().field1.MergeFrom(messages[2]) + generator.group2.add().field1.MergeFrom(messages[0]) + generator.group2.add().field1.MergeFrom(messages[1]) + generator.group2.add().field1.MergeFrom(messages[2]) + + data = generator.SerializeToString() + parsing_merge = unittest_pb2.TestParsingMerge() + parsing_merge.ParseFromString(data) + + # Required and optional fields should be merged. + self.assertEqual(parsing_merge.required_all_types, merged_message) + self.assertEqual(parsing_merge.optional_all_types, merged_message) + self.assertEqual(parsing_merge.optionalgroup.optional_group_all_types, + merged_message) + self.assertEqual(parsing_merge.Extensions[ + unittest_pb2.TestParsingMerge.optional_ext], + merged_message) + + # Repeated fields should not be merged. + self.assertEqual(len(parsing_merge.repeated_all_types), 3) + self.assertEqual(len(parsing_merge.repeatedgroup), 3) + self.assertEqual(len(parsing_merge.Extensions[ + unittest_pb2.TestParsingMerge.repeated_ext]), 3) + + +# Class to test proto3-only features/behavior (updated field presence & enums) +class Proto3Test(basetest.TestCase): + + def testFieldPresence(self): + message = unittest_proto3_arena_pb2.TestAllTypes() + + # We can't test presence of non-repeated, non-submessage fields. + with self.assertRaises(ValueError): + message.HasField("optional_int32") + with self.assertRaises(ValueError): + message.HasField("optional_float") + with self.assertRaises(ValueError): + message.HasField("optional_string") + with self.assertRaises(ValueError): + message.HasField("optional_bool") + + # But we can still test presence of submessage fields. + self.assertFalse(message.HasField("optional_nested_message")) + + # As with proto2, we can't test presence of fields that don't exist, or + # repeated fields. + with self.assertRaises(ValueError): + message.HasField("field_doesnt_exist") + + with self.assertRaises(ValueError): + message.HasField("repeated_int32") + with self.assertRaises(ValueError): + message.HasField("repeated_nested_message") + + # Fields should default to their type-specific default. + self.assertEqual(0, message.optional_int32) + self.assertEqual(0, message.optional_float) + self.assertEqual("", message.optional_string) + self.assertEqual(False, message.optional_bool) + self.assertEqual(0, message.optional_nested_message.bb) + + # Setting a submessage should still return proper presence information. + message.optional_nested_message.bb = 0 + self.assertTrue(message.HasField("optional_nested_message")) + + # Set the fields to non-default values. + message.optional_int32 = 5 + message.optional_float = 1.1 + message.optional_string = "abc" + message.optional_bool = True + message.optional_nested_message.bb = 15 + + # Clearing the fields unsets them and resets their value to default. + message.ClearField("optional_int32") + message.ClearField("optional_float") + message.ClearField("optional_string") + message.ClearField("optional_bool") + message.ClearField("optional_nested_message") + + self.assertEqual(0, message.optional_int32) + self.assertEqual(0, message.optional_float) + self.assertEqual("", message.optional_string) + self.assertEqual(False, message.optional_bool) + self.assertEqual(0, message.optional_nested_message.bb) + + def testAssignUnknownEnum(self): + """Assigning an unknown enum value is allowed and preserves the value.""" + m = unittest_proto3_arena_pb2.TestAllTypes() + + m.optional_nested_enum = 1234567 + self.assertEqual(1234567, m.optional_nested_enum) + m.repeated_nested_enum.append(22334455) + self.assertEqual(22334455, m.repeated_nested_enum[0]) + # Assignment is a different code path than append for the C++ impl. + m.repeated_nested_enum[0] = 7654321 + self.assertEqual(7654321, m.repeated_nested_enum[0]) + serialized = m.SerializeToString() + + m2 = unittest_proto3_arena_pb2.TestAllTypes() + m2.ParseFromString(serialized) + self.assertEqual(1234567, m2.optional_nested_enum) + self.assertEqual(7654321, m2.repeated_nested_enum[0]) + class ValidTypeNamesTest(basetest.TestCase): diff --git a/python/google/protobuf/internal/python_message.py b/python/google/protobuf/internal/python_message.py index 6fda6ae0..6ad0f90d 100755 --- a/python/google/protobuf/internal/python_message.py +++ b/python/google/protobuf/internal/python_message.py @@ -219,12 +219,20 @@ def _AttachFieldHelpers(cls, field_descriptor): def AddDecoder(wiretype, is_packed): tag_bytes = encoder.TagBytes(field_descriptor.number, wiretype) - cls._decoders_by_tag[tag_bytes] = ( - type_checkers.TYPE_TO_DECODER[field_descriptor.type]( - field_descriptor.number, is_repeated, is_packed, - field_descriptor, field_descriptor._default_constructor), - field_descriptor if field_descriptor.containing_oneof is not None - else None) + decode_type = field_descriptor.type + if (decode_type == _FieldDescriptor.TYPE_ENUM and + type_checkers.SupportsOpenEnums(field_descriptor)): + decode_type = _FieldDescriptor.TYPE_INT32 + + oneof_descriptor = None + if field_descriptor.containing_oneof is not None: + oneof_descriptor = field_descriptor + + field_decoder = type_checkers.TYPE_TO_DECODER[decode_type]( + field_descriptor.number, is_repeated, is_packed, + field_descriptor, field_descriptor._default_constructor) + + cls._decoders_by_tag[tag_bytes] = (field_decoder, oneof_descriptor) AddDecoder(type_checkers.FIELD_TYPE_TO_WIRE_TYPE[field_descriptor.type], False) @@ -296,6 +304,8 @@ def _DefaultValueConstructorForField(field): def MakeSubMessageDefault(message): result = message_type._concrete_class() result._SetListener(message._listener_for_children) + if field.containing_oneof: + message._UpdateOneofState(field) return result return MakeSubMessageDefault @@ -476,6 +486,7 @@ def _AddPropertiesForNonRepeatedScalarField(field, cls): type_checker = type_checkers.GetTypeChecker(field) default_value = field.default_value valid_values = set() + is_proto3 = field.containing_type.syntax == "proto3" def getter(self): # TODO(protobuf-team): This may be broken since there may not be @@ -483,15 +494,24 @@ def _AddPropertiesForNonRepeatedScalarField(field, cls): return self._fields.get(field, default_value) getter.__module__ = None getter.__doc__ = 'Getter for %s.' % proto_field_name + + clear_when_set_to_default = is_proto3 and not field.containing_oneof + def field_setter(self, new_value): # pylint: disable=protected-access - self._fields[field] = type_checker.CheckValue(new_value) + # Testing the value for truthiness captures all of the proto3 defaults + # (0, 0.0, enum 0, and False). + new_value = type_checker.CheckValue(new_value) + if clear_when_set_to_default and not new_value: + self._fields.pop(field, None) + else: + self._fields[field] = new_value # Check _cached_byte_size_dirty inline to improve performance, since scalar # setters are called frequently. if not self._cached_byte_size_dirty: self._Modified() - if field.containing_oneof is not None: + if field.containing_oneof: def setter(self, new_value): field_setter(self, new_value) self._UpdateOneofState(field) @@ -624,24 +644,35 @@ def _AddListFieldsMethod(message_descriptor, cls): cls.ListFields = ListFields +_Proto3HasError = 'Protocol message has no non-repeated submessage field "%s"' +_Proto2HasError = 'Protocol message has no non-repeated field "%s"' def _AddHasFieldMethod(message_descriptor, cls): """Helper for _AddMessageMethods().""" - singular_fields = {} + is_proto3 = (message_descriptor.syntax == "proto3") + error_msg = _Proto3HasError if is_proto3 else _Proto2HasError + + hassable_fields = {} for field in message_descriptor.fields: - if field.label != _FieldDescriptor.LABEL_REPEATED: - singular_fields[field.name] = field - # Fields inside oneofs are never repeated (enforced by the compiler). - for field in message_descriptor.oneofs: - singular_fields[field.name] = field + if field.label == _FieldDescriptor.LABEL_REPEATED: + continue + # For proto3, only submessages and fields inside a oneof have presence. + if (is_proto3 and field.cpp_type != _FieldDescriptor.CPPTYPE_MESSAGE and + not field.containing_oneof): + continue + hassable_fields[field.name] = field + + if not is_proto3: + # Fields inside oneofs are never repeated (enforced by the compiler). + for oneof in message_descriptor.oneofs: + hassable_fields[oneof.name] = oneof def HasField(self, field_name): try: - field = singular_fields[field_name] + field = hassable_fields[field_name] except KeyError: - raise ValueError( - 'Protocol message has no singular "%s" field.' % field_name) + raise ValueError(error_msg % field_name) if isinstance(field, descriptor_mod.OneofDescriptor): try: @@ -871,6 +902,7 @@ def _AddMergeFromStringMethod(message_descriptor, cls): local_ReadTag = decoder.ReadTag local_SkipField = decoder.SkipField decoders_by_tag = cls._decoders_by_tag + is_proto3 = message_descriptor.syntax == "proto3" def InternalParse(self, buffer, pos, end): self._Modified() @@ -884,9 +916,11 @@ def _AddMergeFromStringMethod(message_descriptor, cls): new_pos = local_SkipField(buffer, new_pos, end, tag_bytes) if new_pos == -1: return pos - if not unknown_field_list: - unknown_field_list = self._unknown_fields = [] - unknown_field_list.append((tag_bytes, buffer[value_start_pos:new_pos])) + if not is_proto3: + if not unknown_field_list: + unknown_field_list = self._unknown_fields = [] + unknown_field_list.append( + (tag_bytes, buffer[value_start_pos:new_pos])) pos = new_pos else: pos = field_decoder(buffer, new_pos, end, self, field_dict) @@ -1008,6 +1042,8 @@ def _AddMergeFromMethod(cls): # Construct a new object to represent this field. field_value = field._default_constructor(self) fields[field] = field_value + if field.containing_oneof: + self._UpdateOneofState(field) field_value.MergeFrom(value) else: self._fields[field] = value @@ -1252,11 +1288,10 @@ class _ExtensionDict(object): # It's slightly wasteful to lookup the type checker each time, # but we expect this to be a vanishingly uncommon case anyway. - type_checker = type_checkers.GetTypeChecker( - extension_handle) + type_checker = type_checkers.GetTypeChecker(extension_handle) # pylint: disable=protected-access self._extended_message._fields[extension_handle] = ( - type_checker.CheckValue(value)) + type_checker.CheckValue(value)) self._extended_message._Modified() def _FindExtensionByName(self, name): diff --git a/python/google/protobuf/internal/reflection_test.py b/python/google/protobuf/internal/reflection_test.py index 6b24b092..a62d9845 100755 --- a/python/google/protobuf/internal/reflection_test.py +++ b/python/google/protobuf/internal/reflection_test.py @@ -1792,6 +1792,17 @@ class ReflectionTest(basetest.TestCase): # Just check the default value. self.assertEqual(57, msg.inner.value) + @basetest.unittest.skipIf( + api_implementation.Type() != 'cpp' or api_implementation.Version() != 2, + 'CPPv2-specific test') + def testBadArguments(self): + # Some of these assertions used to segfault. + from google.protobuf.pyext import _message + self.assertRaises(TypeError, _message.Message._GetFieldDescriptor, 3) + self.assertRaises(TypeError, _message.Message._GetExtensionDescriptor, 42) + self.assertRaises(TypeError, + unittest_pb2.TestAllTypes().__getattribute__, 42) + # Since we had so many tests for protocol buffer equality, we broke these out # into separate TestCase classes. diff --git a/python/google/protobuf/internal/test_util.py b/python/google/protobuf/internal/test_util.py index 787f4650..d84e3836 100755 --- a/python/google/protobuf/internal/test_util.py +++ b/python/google/protobuf/internal/test_util.py @@ -40,13 +40,19 @@ import os.path from google.protobuf import unittest_import_pb2 from google.protobuf import unittest_pb2 +from google.protobuf import descriptor_pb2 +# Tests whether the given TestAllTypes message is proto2 or not. +# This is used to gate several fields/features that only exist +# for the proto2 version of the message. +def IsProto2(message): + return message.DESCRIPTOR.syntax == "proto2" def SetAllNonLazyFields(message): """Sets every non-lazy field in the message to a unique value. Args: - message: A unittest_pb2.TestAllTypes instance. + message: A TestAllTypes instance. """ # @@ -77,7 +83,8 @@ def SetAllNonLazyFields(message): message.optional_nested_enum = unittest_pb2.TestAllTypes.BAZ message.optional_foreign_enum = unittest_pb2.FOREIGN_BAZ - message.optional_import_enum = unittest_import_pb2.IMPORT_BAZ + if IsProto2(message): + message.optional_import_enum = unittest_import_pb2.IMPORT_BAZ message.optional_string_piece = u'124' message.optional_cord = u'125' @@ -110,7 +117,8 @@ def SetAllNonLazyFields(message): message.repeated_nested_enum.append(unittest_pb2.TestAllTypes.BAR) message.repeated_foreign_enum.append(unittest_pb2.FOREIGN_BAR) - message.repeated_import_enum.append(unittest_import_pb2.IMPORT_BAR) + if IsProto2(message): + message.repeated_import_enum.append(unittest_import_pb2.IMPORT_BAR) message.repeated_string_piece.append(u'224') message.repeated_cord.append(u'225') @@ -140,7 +148,8 @@ def SetAllNonLazyFields(message): message.repeated_nested_enum.append(unittest_pb2.TestAllTypes.BAZ) message.repeated_foreign_enum.append(unittest_pb2.FOREIGN_BAZ) - message.repeated_import_enum.append(unittest_import_pb2.IMPORT_BAZ) + if IsProto2(message): + message.repeated_import_enum.append(unittest_import_pb2.IMPORT_BAZ) message.repeated_string_piece.append(u'324') message.repeated_cord.append(u'325') @@ -149,28 +158,29 @@ def SetAllNonLazyFields(message): # Fields that have defaults. # - message.default_int32 = 401 - message.default_int64 = 402 - message.default_uint32 = 403 - message.default_uint64 = 404 - message.default_sint32 = 405 - message.default_sint64 = 406 - message.default_fixed32 = 407 - message.default_fixed64 = 408 - message.default_sfixed32 = 409 - message.default_sfixed64 = 410 - message.default_float = 411 - message.default_double = 412 - message.default_bool = False - message.default_string = '415' - message.default_bytes = b'416' - - message.default_nested_enum = unittest_pb2.TestAllTypes.FOO - message.default_foreign_enum = unittest_pb2.FOREIGN_FOO - message.default_import_enum = unittest_import_pb2.IMPORT_FOO - - message.default_string_piece = '424' - message.default_cord = '425' + if IsProto2(message): + message.default_int32 = 401 + message.default_int64 = 402 + message.default_uint32 = 403 + message.default_uint64 = 404 + message.default_sint32 = 405 + message.default_sint64 = 406 + message.default_fixed32 = 407 + message.default_fixed64 = 408 + message.default_sfixed32 = 409 + message.default_sfixed64 = 410 + message.default_float = 411 + message.default_double = 412 + message.default_bool = False + message.default_string = '415' + message.default_bytes = b'416' + + message.default_nested_enum = unittest_pb2.TestAllTypes.FOO + message.default_foreign_enum = unittest_pb2.FOREIGN_FOO + message.default_import_enum = unittest_import_pb2.IMPORT_FOO + + message.default_string_piece = '424' + message.default_cord = '425' message.oneof_uint32 = 601 message.oneof_nested_message.bb = 602 @@ -398,7 +408,8 @@ def ExpectAllFieldsSet(test_case, message): test_case.assertTrue(message.HasField('optional_nested_enum')) test_case.assertTrue(message.HasField('optional_foreign_enum')) - test_case.assertTrue(message.HasField('optional_import_enum')) + if IsProto2(message): + test_case.assertTrue(message.HasField('optional_import_enum')) test_case.assertTrue(message.HasField('optional_string_piece')) test_case.assertTrue(message.HasField('optional_cord')) @@ -430,8 +441,9 @@ def ExpectAllFieldsSet(test_case, message): message.optional_nested_enum) test_case.assertEqual(unittest_pb2.FOREIGN_BAZ, message.optional_foreign_enum) - test_case.assertEqual(unittest_import_pb2.IMPORT_BAZ, - message.optional_import_enum) + if IsProto2(message): + test_case.assertEqual(unittest_import_pb2.IMPORT_BAZ, + message.optional_import_enum) # ----------------------------------------------------------------- @@ -457,7 +469,8 @@ def ExpectAllFieldsSet(test_case, message): test_case.assertEqual(2, len(message.repeated_import_message)) test_case.assertEqual(2, len(message.repeated_nested_enum)) test_case.assertEqual(2, len(message.repeated_foreign_enum)) - test_case.assertEqual(2, len(message.repeated_import_enum)) + if IsProto2(message): + test_case.assertEqual(2, len(message.repeated_import_enum)) test_case.assertEqual(2, len(message.repeated_string_piece)) test_case.assertEqual(2, len(message.repeated_cord)) @@ -488,8 +501,9 @@ def ExpectAllFieldsSet(test_case, message): message.repeated_nested_enum[0]) test_case.assertEqual(unittest_pb2.FOREIGN_BAR, message.repeated_foreign_enum[0]) - test_case.assertEqual(unittest_import_pb2.IMPORT_BAR, - message.repeated_import_enum[0]) + if IsProto2(message): + test_case.assertEqual(unittest_import_pb2.IMPORT_BAR, + message.repeated_import_enum[0]) test_case.assertEqual(301, message.repeated_int32[1]) test_case.assertEqual(302, message.repeated_int64[1]) @@ -517,53 +531,55 @@ def ExpectAllFieldsSet(test_case, message): message.repeated_nested_enum[1]) test_case.assertEqual(unittest_pb2.FOREIGN_BAZ, message.repeated_foreign_enum[1]) - test_case.assertEqual(unittest_import_pb2.IMPORT_BAZ, - message.repeated_import_enum[1]) + if IsProto2(message): + test_case.assertEqual(unittest_import_pb2.IMPORT_BAZ, + message.repeated_import_enum[1]) # ----------------------------------------------------------------- - test_case.assertTrue(message.HasField('default_int32')) - test_case.assertTrue(message.HasField('default_int64')) - test_case.assertTrue(message.HasField('default_uint32')) - test_case.assertTrue(message.HasField('default_uint64')) - test_case.assertTrue(message.HasField('default_sint32')) - test_case.assertTrue(message.HasField('default_sint64')) - test_case.assertTrue(message.HasField('default_fixed32')) - test_case.assertTrue(message.HasField('default_fixed64')) - test_case.assertTrue(message.HasField('default_sfixed32')) - test_case.assertTrue(message.HasField('default_sfixed64')) - test_case.assertTrue(message.HasField('default_float')) - test_case.assertTrue(message.HasField('default_double')) - test_case.assertTrue(message.HasField('default_bool')) - test_case.assertTrue(message.HasField('default_string')) - test_case.assertTrue(message.HasField('default_bytes')) - - test_case.assertTrue(message.HasField('default_nested_enum')) - test_case.assertTrue(message.HasField('default_foreign_enum')) - test_case.assertTrue(message.HasField('default_import_enum')) - - test_case.assertEqual(401, message.default_int32) - test_case.assertEqual(402, message.default_int64) - test_case.assertEqual(403, message.default_uint32) - test_case.assertEqual(404, message.default_uint64) - test_case.assertEqual(405, message.default_sint32) - test_case.assertEqual(406, message.default_sint64) - test_case.assertEqual(407, message.default_fixed32) - test_case.assertEqual(408, message.default_fixed64) - test_case.assertEqual(409, message.default_sfixed32) - test_case.assertEqual(410, message.default_sfixed64) - test_case.assertEqual(411, message.default_float) - test_case.assertEqual(412, message.default_double) - test_case.assertEqual(False, message.default_bool) - test_case.assertEqual('415', message.default_string) - test_case.assertEqual(b'416', message.default_bytes) - - test_case.assertEqual(unittest_pb2.TestAllTypes.FOO, - message.default_nested_enum) - test_case.assertEqual(unittest_pb2.FOREIGN_FOO, - message.default_foreign_enum) - test_case.assertEqual(unittest_import_pb2.IMPORT_FOO, - message.default_import_enum) + if IsProto2(message): + test_case.assertTrue(message.HasField('default_int32')) + test_case.assertTrue(message.HasField('default_int64')) + test_case.assertTrue(message.HasField('default_uint32')) + test_case.assertTrue(message.HasField('default_uint64')) + test_case.assertTrue(message.HasField('default_sint32')) + test_case.assertTrue(message.HasField('default_sint64')) + test_case.assertTrue(message.HasField('default_fixed32')) + test_case.assertTrue(message.HasField('default_fixed64')) + test_case.assertTrue(message.HasField('default_sfixed32')) + test_case.assertTrue(message.HasField('default_sfixed64')) + test_case.assertTrue(message.HasField('default_float')) + test_case.assertTrue(message.HasField('default_double')) + test_case.assertTrue(message.HasField('default_bool')) + test_case.assertTrue(message.HasField('default_string')) + test_case.assertTrue(message.HasField('default_bytes')) + + test_case.assertTrue(message.HasField('default_nested_enum')) + test_case.assertTrue(message.HasField('default_foreign_enum')) + test_case.assertTrue(message.HasField('default_import_enum')) + + test_case.assertEqual(401, message.default_int32) + test_case.assertEqual(402, message.default_int64) + test_case.assertEqual(403, message.default_uint32) + test_case.assertEqual(404, message.default_uint64) + test_case.assertEqual(405, message.default_sint32) + test_case.assertEqual(406, message.default_sint64) + test_case.assertEqual(407, message.default_fixed32) + test_case.assertEqual(408, message.default_fixed64) + test_case.assertEqual(409, message.default_sfixed32) + test_case.assertEqual(410, message.default_sfixed64) + test_case.assertEqual(411, message.default_float) + test_case.assertEqual(412, message.default_double) + test_case.assertEqual(False, message.default_bool) + test_case.assertEqual('415', message.default_string) + test_case.assertEqual(b'416', message.default_bytes) + + test_case.assertEqual(unittest_pb2.TestAllTypes.FOO, + message.default_nested_enum) + test_case.assertEqual(unittest_pb2.FOREIGN_FOO, + message.default_foreign_enum) + test_case.assertEqual(unittest_import_pb2.IMPORT_FOO, + message.default_import_enum) def GoldenFile(filename): @@ -594,7 +610,7 @@ def SetAllPackedFields(message): """Sets every field in the message to a unique value. Args: - message: A unittest_pb2.TestPackedTypes instance. + message: A TestPackedTypes instance. """ message.packed_int32.extend([601, 701]) message.packed_int64.extend([602, 702]) diff --git a/python/google/protobuf/internal/text_format_test.py b/python/google/protobuf/internal/text_format_test.py index 55e3c2c8..b307620d 100755 --- a/python/google/protobuf/internal/text_format_test.py +++ b/python/google/protobuf/internal/text_format_test.py @@ -37,13 +37,17 @@ __author__ = 'kenton@google.com (Kenton Varda)' import re from google.apputils import basetest -from google.protobuf import text_format + +from google.protobuf import unittest_mset_pb2 +from google.protobuf import unittest_pb2 +from google.protobuf import unittest_proto3_arena_pb2 from google.protobuf.internal import api_implementation +from google.protobuf.internal import _parameterized from google.protobuf.internal import test_util -from google.protobuf import unittest_pb2 -from google.protobuf import unittest_mset_pb2 +from google.protobuf import text_format -class TextFormatTest(basetest.TestCase): +# Base class with some common functionality. +class TextFormatBase(basetest.TestCase): def ReadGolden(self, golden_filename): with test_util.GoldenFile(golden_filename) as f: @@ -57,73 +61,24 @@ class TextFormatTest(basetest.TestCase): def CompareToGoldenText(self, text, golden_text): self.assertMultiLineEqual(text, golden_text) - def testPrintAllFields(self): - message = unittest_pb2.TestAllTypes() - test_util.SetAllFields(message) - self.CompareToGoldenFile( - self.RemoveRedundantZeros(text_format.MessageToString(message)), - 'text_format_unittest_data_oneof_implemented.txt') - - def testPrintInIndexOrder(self): - message = unittest_pb2.TestFieldOrderings() - message.my_string = '115' - message.my_int = 101 - message.my_float = 111 - message.optional_nested_message.oo = 0 - message.optional_nested_message.bb = 1 - self.CompareToGoldenText( - self.RemoveRedundantZeros(text_format.MessageToString( - message, use_index_order=True)), - 'my_string: \"115\"\nmy_int: 101\nmy_float: 111\n' - 'optional_nested_message {\n oo: 0\n bb: 1\n}\n') - self.CompareToGoldenText( - self.RemoveRedundantZeros(text_format.MessageToString( - message)), - 'my_int: 101\nmy_string: \"115\"\nmy_float: 111\n' - 'optional_nested_message {\n bb: 1\n oo: 0\n}\n') - - def testPrintAllExtensions(self): - message = unittest_pb2.TestAllExtensions() - test_util.SetAllExtensions(message) - self.CompareToGoldenFile( - self.RemoveRedundantZeros(text_format.MessageToString(message)), - 'text_format_unittest_extensions_data.txt') - - def testPrintAllFieldsPointy(self): - message = unittest_pb2.TestAllTypes() - test_util.SetAllFields(message) - self.CompareToGoldenFile( - self.RemoveRedundantZeros( - text_format.MessageToString(message, pointy_brackets=True)), - 'text_format_unittest_data_pointy_oneof.txt') + def RemoveRedundantZeros(self, text): + # Some platforms print 1e+5 as 1e+005. This is fine, but we need to remove + # these zeros in order to match the golden file. + text = text.replace('e+0','e+').replace('e+0','e+') \ + .replace('e-0','e-').replace('e-0','e-') + # Floating point fields are printed with .0 suffix even if they are + # actualy integer numbers. + text = re.compile('\.0$', re.MULTILINE).sub('', text) + return text - def testPrintAllExtensionsPointy(self): - message = unittest_pb2.TestAllExtensions() - test_util.SetAllExtensions(message) - self.CompareToGoldenFile( - self.RemoveRedundantZeros(text_format.MessageToString( - message, pointy_brackets=True)), - 'text_format_unittest_extensions_data_pointy.txt') - def testPrintMessageSet(self): - message = unittest_mset_pb2.TestMessageSetContainer() - ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension - ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension - message.message_set.Extensions[ext1].i = 23 - message.message_set.Extensions[ext2].str = 'foo' - self.CompareToGoldenText( - text_format.MessageToString(message), - 'message_set {\n' - ' [protobuf_unittest.TestMessageSetExtension1] {\n' - ' i: 23\n' - ' }\n' - ' [protobuf_unittest.TestMessageSetExtension2] {\n' - ' str: \"foo\"\n' - ' }\n' - '}\n') +@_parameterized.Parameters( + (unittest_pb2), + (unittest_proto3_arena_pb2)) +class TextFormatTest(TextFormatBase): - def testPrintExotic(self): - message = unittest_pb2.TestAllTypes() + def testPrintExotic(self, message_module): + message = message_module.TestAllTypes() message.repeated_int64.append(-9223372036854775808) message.repeated_uint64.append(18446744073709551615) message.repeated_double.append(123.456) @@ -142,61 +97,44 @@ class TextFormatTest(basetest.TestCase): ' "\\000\\001\\007\\010\\014\\n\\r\\t\\013\\\\\\\'\\""\n' 'repeated_string: "\\303\\274\\352\\234\\237"\n') - def testPrintExoticUnicodeSubclass(self): + def testPrintExoticUnicodeSubclass(self, message_module): class UnicodeSub(unicode): pass - message = unittest_pb2.TestAllTypes() + message = message_module.TestAllTypes() message.repeated_string.append(UnicodeSub(u'\u00fc\ua71f')) self.CompareToGoldenText( text_format.MessageToString(message), 'repeated_string: "\\303\\274\\352\\234\\237"\n') - def testPrintNestedMessageAsOneLine(self): - message = unittest_pb2.TestAllTypes() + def testPrintNestedMessageAsOneLine(self, message_module): + message = message_module.TestAllTypes() msg = message.repeated_nested_message.add() msg.bb = 42 self.CompareToGoldenText( text_format.MessageToString(message, as_one_line=True), 'repeated_nested_message { bb: 42 }') - def testPrintRepeatedFieldsAsOneLine(self): - message = unittest_pb2.TestAllTypes() + def testPrintRepeatedFieldsAsOneLine(self, message_module): + message = message_module.TestAllTypes() message.repeated_int32.append(1) message.repeated_int32.append(1) message.repeated_int32.append(3) - message.repeated_string.append("Google") - message.repeated_string.append("Zurich") + message.repeated_string.append('Google') + message.repeated_string.append('Zurich') self.CompareToGoldenText( text_format.MessageToString(message, as_one_line=True), 'repeated_int32: 1 repeated_int32: 1 repeated_int32: 3 ' 'repeated_string: "Google" repeated_string: "Zurich"') - def testPrintNestedNewLineInStringAsOneLine(self): - message = unittest_pb2.TestAllTypes() - message.optional_string = "a\nnew\nline" + def testPrintNestedNewLineInStringAsOneLine(self, message_module): + message = message_module.TestAllTypes() + message.optional_string = 'a\nnew\nline' self.CompareToGoldenText( text_format.MessageToString(message, as_one_line=True), 'optional_string: "a\\nnew\\nline"') - def testPrintMessageSetAsOneLine(self): - message = unittest_mset_pb2.TestMessageSetContainer() - ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension - ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension - message.message_set.Extensions[ext1].i = 23 - message.message_set.Extensions[ext2].str = 'foo' - self.CompareToGoldenText( - text_format.MessageToString(message, as_one_line=True), - 'message_set {' - ' [protobuf_unittest.TestMessageSetExtension1] {' - ' i: 23' - ' }' - ' [protobuf_unittest.TestMessageSetExtension2] {' - ' str: \"foo\"' - ' }' - ' }') - - def testPrintExoticAsOneLine(self): - message = unittest_pb2.TestAllTypes() + def testPrintExoticAsOneLine(self, message_module): + message = message_module.TestAllTypes() message.repeated_int64.append(-9223372036854775808) message.repeated_uint64.append(18446744073709551615) message.repeated_double.append(123.456) @@ -216,8 +154,8 @@ class TextFormatTest(basetest.TestCase): '"\\000\\001\\007\\010\\014\\n\\r\\t\\013\\\\\\\'\\""' ' repeated_string: "\\303\\274\\352\\234\\237"') - def testRoundTripExoticAsOneLine(self): - message = unittest_pb2.TestAllTypes() + def testRoundTripExoticAsOneLine(self, message_module): + message = message_module.TestAllTypes() message.repeated_int64.append(-9223372036854775808) message.repeated_uint64.append(18446744073709551615) message.repeated_double.append(123.456) @@ -229,7 +167,7 @@ class TextFormatTest(basetest.TestCase): # Test as_utf8 = False. wire_text = text_format.MessageToString( message, as_one_line=True, as_utf8=False) - parsed_message = unittest_pb2.TestAllTypes() + parsed_message = message_module.TestAllTypes() r = text_format.Parse(wire_text, parsed_message) self.assertIs(r, parsed_message) self.assertEquals(message, parsed_message) @@ -237,25 +175,25 @@ class TextFormatTest(basetest.TestCase): # Test as_utf8 = True. wire_text = text_format.MessageToString( message, as_one_line=True, as_utf8=True) - parsed_message = unittest_pb2.TestAllTypes() + parsed_message = message_module.TestAllTypes() r = text_format.Parse(wire_text, parsed_message) self.assertIs(r, parsed_message) self.assertEquals(message, parsed_message, '\n%s != %s' % (message, parsed_message)) - def testPrintRawUtf8String(self): - message = unittest_pb2.TestAllTypes() + def testPrintRawUtf8String(self, message_module): + message = message_module.TestAllTypes() message.repeated_string.append(u'\u00fc\ua71f') text = text_format.MessageToString(message, as_utf8=True) self.CompareToGoldenText(text, 'repeated_string: "\303\274\352\234\237"\n') - parsed_message = unittest_pb2.TestAllTypes() + parsed_message = message_module.TestAllTypes() text_format.Parse(text, parsed_message) self.assertEquals(message, parsed_message, '\n%s != %s' % (message, parsed_message)) - def testPrintFloatFormat(self): + def testPrintFloatFormat(self, message_module): # Check that float_format argument is passed to sub-message formatting. - message = unittest_pb2.NestedTestAllTypes() + message = message_module.NestedTestAllTypes() # We use 1.25 as it is a round number in binary. The proto 32-bit float # will not gain additional imprecise digits as a 64-bit Python float and # show up in its str. 32-bit 1.2 is noisy when extended to 64-bit: @@ -285,85 +223,24 @@ class TextFormatTest(basetest.TestCase): self.RemoveRedundantZeros(text_message), 'payload {{ {} {} {} {} }}'.format(*formatted_fields)) - def testMessageToString(self): - message = unittest_pb2.ForeignMessage() + def testMessageToString(self, message_module): + message = message_module.ForeignMessage() message.c = 123 self.assertEqual('c: 123\n', str(message)) - def RemoveRedundantZeros(self, text): - # Some platforms print 1e+5 as 1e+005. This is fine, but we need to remove - # these zeros in order to match the golden file. - text = text.replace('e+0','e+').replace('e+0','e+') \ - .replace('e-0','e-').replace('e-0','e-') - # Floating point fields are printed with .0 suffix even if they are - # actualy integer numbers. - text = re.compile('\.0$', re.MULTILINE).sub('', text) - return text - - def testParseGolden(self): - golden_text = '\n'.join(self.ReadGolden('text_format_unittest_data.txt')) - parsed_message = unittest_pb2.TestAllTypes() - r = text_format.Parse(golden_text, parsed_message) - self.assertIs(r, parsed_message) - - message = unittest_pb2.TestAllTypes() - test_util.SetAllFields(message) - self.assertEquals(message, parsed_message) - - def testParseGoldenExtensions(self): - golden_text = '\n'.join(self.ReadGolden( - 'text_format_unittest_extensions_data.txt')) - parsed_message = unittest_pb2.TestAllExtensions() - text_format.Parse(golden_text, parsed_message) - - message = unittest_pb2.TestAllExtensions() - test_util.SetAllExtensions(message) - self.assertEquals(message, parsed_message) - - def testParseAllFields(self): - message = unittest_pb2.TestAllTypes() + def testParseAllFields(self, message_module): + message = message_module.TestAllTypes() test_util.SetAllFields(message) ascii_text = text_format.MessageToString(message) - parsed_message = unittest_pb2.TestAllTypes() - text_format.Parse(ascii_text, parsed_message) - self.assertEqual(message, parsed_message) - test_util.ExpectAllFieldsSet(self, message) - - def testParseAllExtensions(self): - message = unittest_pb2.TestAllExtensions() - test_util.SetAllExtensions(message) - ascii_text = text_format.MessageToString(message) - - parsed_message = unittest_pb2.TestAllExtensions() + parsed_message = message_module.TestAllTypes() text_format.Parse(ascii_text, parsed_message) self.assertEqual(message, parsed_message) + if message_module is unittest_pb2: + test_util.ExpectAllFieldsSet(self, message) - def testParseMessageSet(self): - message = unittest_pb2.TestAllTypes() - text = ('repeated_uint64: 1\n' - 'repeated_uint64: 2\n') - text_format.Parse(text, message) - self.assertEqual(1, message.repeated_uint64[0]) - self.assertEqual(2, message.repeated_uint64[1]) - - message = unittest_mset_pb2.TestMessageSetContainer() - text = ('message_set {\n' - ' [protobuf_unittest.TestMessageSetExtension1] {\n' - ' i: 23\n' - ' }\n' - ' [protobuf_unittest.TestMessageSetExtension2] {\n' - ' str: \"foo\"\n' - ' }\n' - '}\n') - text_format.Parse(text, message) - ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension - ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension - self.assertEquals(23, message.message_set.Extensions[ext1].i) - self.assertEquals('foo', message.message_set.Extensions[ext2].str) - - def testParseExotic(self): - message = unittest_pb2.TestAllTypes() + def testParseExotic(self, message_module): + message = message_module.TestAllTypes() text = ('repeated_int64: -9223372036854775808\n' 'repeated_uint64: 18446744073709551615\n' 'repeated_double: 123.456\n' @@ -388,8 +265,8 @@ class TextFormatTest(basetest.TestCase): self.assertEqual(u'\u00fc\ua71f', message.repeated_string[2]) self.assertEqual(u'\u00fc', message.repeated_string[3]) - def testParseTrailingCommas(self): - message = unittest_pb2.TestAllTypes() + def testParseTrailingCommas(self, message_module): + message = message_module.TestAllTypes() text = ('repeated_int64: 100;\n' 'repeated_int64: 200;\n' 'repeated_int64: 300,\n' @@ -403,51 +280,37 @@ class TextFormatTest(basetest.TestCase): self.assertEqual(u'one', message.repeated_string[0]) self.assertEqual(u'two', message.repeated_string[1]) - def testParseEmptyText(self): - message = unittest_pb2.TestAllTypes() + def testParseEmptyText(self, message_module): + message = message_module.TestAllTypes() text = '' text_format.Parse(text, message) - self.assertEquals(unittest_pb2.TestAllTypes(), message) + self.assertEquals(message_module.TestAllTypes(), message) - def testParseInvalidUtf8(self): - message = unittest_pb2.TestAllTypes() + def testParseInvalidUtf8(self, message_module): + message = message_module.TestAllTypes() text = 'repeated_string: "\\xc3\\xc3"' self.assertRaises(text_format.ParseError, text_format.Parse, text, message) - def testParseSingleWord(self): - message = unittest_pb2.TestAllTypes() + def testParseSingleWord(self, message_module): + message = message_module.TestAllTypes() text = 'foo' - self.assertRaisesWithLiteralMatch( + self.assertRaisesRegexp( text_format.ParseError, - ('1:1 : Message type "protobuf_unittest.TestAllTypes" has no field named ' - '"foo".'), + (r'1:1 : Message type "\w+.TestAllTypes" has no field named ' + r'"foo".'), text_format.Parse, text, message) - def testParseUnknownField(self): - message = unittest_pb2.TestAllTypes() + def testParseUnknownField(self, message_module): + message = message_module.TestAllTypes() text = 'unknown_field: 8\n' - self.assertRaisesWithLiteralMatch( + self.assertRaisesRegexp( text_format.ParseError, - ('1:1 : Message type "protobuf_unittest.TestAllTypes" has no field named ' - '"unknown_field".'), + (r'1:1 : Message type "\w+.TestAllTypes" has no field named ' + r'"unknown_field".'), text_format.Parse, text, message) - def testParseBadExtension(self): - message = unittest_pb2.TestAllExtensions() - text = '[unknown_extension]: 8\n' - self.assertRaisesWithLiteralMatch( - text_format.ParseError, - '1:2 : Extension "unknown_extension" not registered.', - text_format.Parse, text, message) - message = unittest_pb2.TestAllTypes() - self.assertRaisesWithLiteralMatch( - text_format.ParseError, - ('1:2 : Message type "protobuf_unittest.TestAllTypes" does not have ' - 'extensions.'), - text_format.Parse, text, message) - - def testParseGroupNotClosed(self): - message = unittest_pb2.TestAllTypes() + def testParseGroupNotClosed(self, message_module): + message = message_module.TestAllTypes() text = 'RepeatedGroup: <' self.assertRaisesWithLiteralMatch( text_format.ParseError, '1:16 : Expected ">".', @@ -458,46 +321,46 @@ class TextFormatTest(basetest.TestCase): text_format.ParseError, '1:16 : Expected "}".', text_format.Parse, text, message) - def testParseEmptyGroup(self): - message = unittest_pb2.TestAllTypes() + def testParseEmptyGroup(self, message_module): + message = message_module.TestAllTypes() text = 'OptionalGroup: {}' text_format.Parse(text, message) self.assertTrue(message.HasField('optionalgroup')) message.Clear() - message = unittest_pb2.TestAllTypes() + message = message_module.TestAllTypes() text = 'OptionalGroup: <>' text_format.Parse(text, message) self.assertTrue(message.HasField('optionalgroup')) - def testParseBadEnumValue(self): - message = unittest_pb2.TestAllTypes() + def testParseBadEnumValue(self, message_module): + message = message_module.TestAllTypes() text = 'optional_nested_enum: BARR' - self.assertRaisesWithLiteralMatch( + self.assertRaisesRegexp( text_format.ParseError, - ('1:23 : Enum type "protobuf_unittest.TestAllTypes.NestedEnum" ' - 'has no value named BARR.'), + (r'1:23 : Enum type "\w+.TestAllTypes.NestedEnum" ' + r'has no value named BARR.'), text_format.Parse, text, message) - message = unittest_pb2.TestAllTypes() + message = message_module.TestAllTypes() text = 'optional_nested_enum: 100' - self.assertRaisesWithLiteralMatch( + self.assertRaisesRegexp( text_format.ParseError, - ('1:23 : Enum type "protobuf_unittest.TestAllTypes.NestedEnum" ' - 'has no value with number 100.'), + (r'1:23 : Enum type "\w+.TestAllTypes.NestedEnum" ' + r'has no value with number 100.'), text_format.Parse, text, message) - def testParseBadIntValue(self): - message = unittest_pb2.TestAllTypes() + def testParseBadIntValue(self, message_module): + message = message_module.TestAllTypes() text = 'optional_int32: bork' self.assertRaisesWithLiteralMatch( text_format.ParseError, ('1:17 : Couldn\'t parse integer: bork'), text_format.Parse, text, message) - def testParseStringFieldUnescape(self): - message = unittest_pb2.TestAllTypes() + def testParseStringFieldUnescape(self, message_module): + message = message_module.TestAllTypes() text = r'''repeated_string: "\xf\x62" repeated_string: "\\xf\\x62" repeated_string: "\\\xf\\\x62" @@ -516,40 +379,205 @@ class TextFormatTest(basetest.TestCase): message.repeated_string[4]) self.assertEqual(SLASH + 'x20', message.repeated_string[5]) - def testMergeDuplicateScalars(self): - message = unittest_pb2.TestAllTypes() + def testMergeDuplicateScalars(self, message_module): + message = message_module.TestAllTypes() text = ('optional_int32: 42 ' 'optional_int32: 67') r = text_format.Merge(text, message) self.assertIs(r, message) self.assertEqual(67, message.optional_int32) - def testParseDuplicateScalars(self): - message = unittest_pb2.TestAllTypes() - text = ('optional_int32: 42 ' - 'optional_int32: 67') - self.assertRaisesWithLiteralMatch( - text_format.ParseError, - ('1:36 : Message type "protobuf_unittest.TestAllTypes" should not ' - 'have multiple "optional_int32" fields.'), - text_format.Parse, text, message) - - def testMergeDuplicateNestedMessageScalars(self): - message = unittest_pb2.TestAllTypes() + def testMergeDuplicateNestedMessageScalars(self, message_module): + message = message_module.TestAllTypes() text = ('optional_nested_message { bb: 1 } ' 'optional_nested_message { bb: 2 }') r = text_format.Merge(text, message) self.assertTrue(r is message) self.assertEqual(2, message.optional_nested_message.bb) - def testParseDuplicateNestedMessageScalars(self): + def testParseOneof(self, message_module): + m = message_module.TestAllTypes() + m.oneof_uint32 = 11 + m2 = message_module.TestAllTypes() + text_format.Parse(text_format.MessageToString(m), m2) + self.assertEqual('oneof_uint32', m2.WhichOneof('oneof_field')) + + +# These are tests that aren't fundamentally specific to proto2, but are at +# the moment because of differences between the proto2 and proto3 test schemas. +# Ideally the schemas would be made more similar so these tests could pass. +class OnlyWorksWithProto2RightNowTests(TextFormatBase): + + def testParseGolden(self): + golden_text = '\n'.join(self.ReadGolden('text_format_unittest_data.txt')) + parsed_message = unittest_pb2.TestAllTypes() + r = text_format.Parse(golden_text, parsed_message) + self.assertIs(r, parsed_message) + message = unittest_pb2.TestAllTypes() - text = ('optional_nested_message { bb: 1 } ' - 'optional_nested_message { bb: 2 }') + test_util.SetAllFields(message) + self.assertEquals(message, parsed_message) + + def testPrintAllFields(self): + message = unittest_pb2.TestAllTypes() + test_util.SetAllFields(message) + self.CompareToGoldenFile( + self.RemoveRedundantZeros(text_format.MessageToString(message)), + 'text_format_unittest_data_oneof_implemented.txt') + + def testPrintAllFieldsPointy(self): + message = unittest_pb2.TestAllTypes() + test_util.SetAllFields(message) + self.CompareToGoldenFile( + self.RemoveRedundantZeros( + text_format.MessageToString(message, pointy_brackets=True)), + 'text_format_unittest_data_pointy_oneof.txt') + + def testPrintInIndexOrder(self): + message = unittest_pb2.TestFieldOrderings() + message.my_string = '115' + message.my_int = 101 + message.my_float = 111 + message.optional_nested_message.oo = 0 + message.optional_nested_message.bb = 1 + self.CompareToGoldenText( + self.RemoveRedundantZeros(text_format.MessageToString( + message, use_index_order=True)), + 'my_string: \"115\"\nmy_int: 101\nmy_float: 111\n' + 'optional_nested_message {\n oo: 0\n bb: 1\n}\n') + self.CompareToGoldenText( + self.RemoveRedundantZeros(text_format.MessageToString( + message)), + 'my_int: 101\nmy_string: \"115\"\nmy_float: 111\n' + 'optional_nested_message {\n bb: 1\n oo: 0\n}\n') + + def testMergeLinesGolden(self): + opened = self.ReadGolden('text_format_unittest_data.txt') + parsed_message = unittest_pb2.TestAllTypes() + r = text_format.MergeLines(opened, parsed_message) + self.assertIs(r, parsed_message) + + message = unittest_pb2.TestAllTypes() + test_util.SetAllFields(message) + self.assertEqual(message, parsed_message) + + def testParseLinesGolden(self): + opened = self.ReadGolden('text_format_unittest_data.txt') + parsed_message = unittest_pb2.TestAllTypes() + r = text_format.ParseLines(opened, parsed_message) + self.assertIs(r, parsed_message) + + message = unittest_pb2.TestAllTypes() + test_util.SetAllFields(message) + self.assertEquals(message, parsed_message) + + +# Tests of proto2-only features (MessageSet and extensions). +class Proto2Tests(TextFormatBase): + + def testPrintMessageSet(self): + message = unittest_mset_pb2.TestMessageSetContainer() + ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension + ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension + message.message_set.Extensions[ext1].i = 23 + message.message_set.Extensions[ext2].str = 'foo' + self.CompareToGoldenText( + text_format.MessageToString(message), + 'message_set {\n' + ' [protobuf_unittest.TestMessageSetExtension1] {\n' + ' i: 23\n' + ' }\n' + ' [protobuf_unittest.TestMessageSetExtension2] {\n' + ' str: \"foo\"\n' + ' }\n' + '}\n') + + def testPrintMessageSetAsOneLine(self): + message = unittest_mset_pb2.TestMessageSetContainer() + ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension + ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension + message.message_set.Extensions[ext1].i = 23 + message.message_set.Extensions[ext2].str = 'foo' + self.CompareToGoldenText( + text_format.MessageToString(message, as_one_line=True), + 'message_set {' + ' [protobuf_unittest.TestMessageSetExtension1] {' + ' i: 23' + ' }' + ' [protobuf_unittest.TestMessageSetExtension2] {' + ' str: \"foo\"' + ' }' + ' }') + + def testParseMessageSet(self): + message = unittest_pb2.TestAllTypes() + text = ('repeated_uint64: 1\n' + 'repeated_uint64: 2\n') + text_format.Parse(text, message) + self.assertEqual(1, message.repeated_uint64[0]) + self.assertEqual(2, message.repeated_uint64[1]) + + message = unittest_mset_pb2.TestMessageSetContainer() + text = ('message_set {\n' + ' [protobuf_unittest.TestMessageSetExtension1] {\n' + ' i: 23\n' + ' }\n' + ' [protobuf_unittest.TestMessageSetExtension2] {\n' + ' str: \"foo\"\n' + ' }\n' + '}\n') + text_format.Parse(text, message) + ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension + ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension + self.assertEquals(23, message.message_set.Extensions[ext1].i) + self.assertEquals('foo', message.message_set.Extensions[ext2].str) + + def testPrintAllExtensions(self): + message = unittest_pb2.TestAllExtensions() + test_util.SetAllExtensions(message) + self.CompareToGoldenFile( + self.RemoveRedundantZeros(text_format.MessageToString(message)), + 'text_format_unittest_extensions_data.txt') + + def testPrintAllExtensionsPointy(self): + message = unittest_pb2.TestAllExtensions() + test_util.SetAllExtensions(message) + self.CompareToGoldenFile( + self.RemoveRedundantZeros(text_format.MessageToString( + message, pointy_brackets=True)), + 'text_format_unittest_extensions_data_pointy.txt') + + def testParseGoldenExtensions(self): + golden_text = '\n'.join(self.ReadGolden( + 'text_format_unittest_extensions_data.txt')) + parsed_message = unittest_pb2.TestAllExtensions() + text_format.Parse(golden_text, parsed_message) + + message = unittest_pb2.TestAllExtensions() + test_util.SetAllExtensions(message) + self.assertEquals(message, parsed_message) + + def testParseAllExtensions(self): + message = unittest_pb2.TestAllExtensions() + test_util.SetAllExtensions(message) + ascii_text = text_format.MessageToString(message) + + parsed_message = unittest_pb2.TestAllExtensions() + text_format.Parse(ascii_text, parsed_message) + self.assertEqual(message, parsed_message) + + def testParseBadExtension(self): + message = unittest_pb2.TestAllExtensions() + text = '[unknown_extension]: 8\n' self.assertRaisesWithLiteralMatch( text_format.ParseError, - ('1:65 : Message type "protobuf_unittest.TestAllTypes.NestedMessage" ' - 'should not have multiple "bb" fields.'), + '1:2 : Extension "unknown_extension" not registered.', + text_format.Parse, text, message) + message = unittest_pb2.TestAllTypes() + self.assertRaisesWithLiteralMatch( + text_format.ParseError, + ('1:2 : Message type "protobuf_unittest.TestAllTypes" does not have ' + 'extensions.'), text_format.Parse, text, message) def testMergeDuplicateExtensionScalars(self): @@ -572,32 +600,25 @@ class TextFormatTest(basetest.TestCase): '"protobuf_unittest.optional_int32_extension" extensions.'), text_format.Parse, text, message) - def testParseLinesGolden(self): - opened = self.ReadGolden('text_format_unittest_data.txt') - parsed_message = unittest_pb2.TestAllTypes() - r = text_format.ParseLines(opened, parsed_message) - self.assertIs(r, parsed_message) - + def testParseDuplicateNestedMessageScalars(self): message = unittest_pb2.TestAllTypes() - test_util.SetAllFields(message) - self.assertEquals(message, parsed_message) - - def testMergeLinesGolden(self): - opened = self.ReadGolden('text_format_unittest_data.txt') - parsed_message = unittest_pb2.TestAllTypes() - r = text_format.MergeLines(opened, parsed_message) - self.assertIs(r, parsed_message) + text = ('optional_nested_message { bb: 1 } ' + 'optional_nested_message { bb: 2 }') + self.assertRaisesWithLiteralMatch( + text_format.ParseError, + ('1:65 : Message type "protobuf_unittest.TestAllTypes.NestedMessage" ' + 'should not have multiple "bb" fields.'), + text_format.Parse, text, message) + def testParseDuplicateScalars(self): message = unittest_pb2.TestAllTypes() - test_util.SetAllFields(message) - self.assertEqual(message, parsed_message) - - def testParseOneof(self): - m = unittest_pb2.TestAllTypes() - m.oneof_uint32 = 11 - m2 = unittest_pb2.TestAllTypes() - text_format.Parse(text_format.MessageToString(m), m2) - self.assertEqual('oneof_uint32', m2.WhichOneof('oneof_field')) + text = ('optional_int32: 42 ' + 'optional_int32: 67') + self.assertRaisesWithLiteralMatch( + text_format.ParseError, + ('1:36 : Message type "protobuf_unittest.TestAllTypes" should not ' + 'have multiple "optional_int32" fields.'), + text_format.Parse, text, message) class TokenizerTest(basetest.TestCase): diff --git a/python/google/protobuf/internal/type_checkers.py b/python/google/protobuf/internal/type_checkers.py index 118725da..76c056c4 100755 --- a/python/google/protobuf/internal/type_checkers.py +++ b/python/google/protobuf/internal/type_checkers.py @@ -59,6 +59,8 @@ from google.protobuf import descriptor _FieldDescriptor = descriptor.FieldDescriptor +def SupportsOpenEnums(field_descriptor): + return field_descriptor.containing_type.syntax == "proto3" def GetTypeChecker(field): """Returns a type checker for a message field of the specified types. @@ -74,7 +76,11 @@ def GetTypeChecker(field): field.type == _FieldDescriptor.TYPE_STRING): return UnicodeValueChecker() if field.cpp_type == _FieldDescriptor.CPPTYPE_ENUM: - return EnumValueChecker(field.enum_type) + if SupportsOpenEnums(field): + # When open enums are supported, any int32 can be assigned. + return _VALUE_CHECKERS[_FieldDescriptor.CPPTYPE_INT32] + else: + return EnumValueChecker(field.enum_type) return _VALUE_CHECKERS[field.cpp_type] diff --git a/python/google/protobuf/internal/unknown_fields_test.py b/python/google/protobuf/internal/unknown_fields_test.py index a4dc1f7c..59f9ae4c 100755 --- a/python/google/protobuf/internal/unknown_fields_test.py +++ b/python/google/protobuf/internal/unknown_fields_test.py @@ -38,6 +38,7 @@ __author__ = 'bohdank@google.com (Bohdan Koval)' from google.apputils import basetest from google.protobuf import unittest_mset_pb2 from google.protobuf import unittest_pb2 +from google.protobuf import unittest_proto3_arena_pb2 from google.protobuf.internal import api_implementation from google.protobuf.internal import encoder from google.protobuf.internal import missing_enum_values_pb2 @@ -45,10 +46,81 @@ from google.protobuf.internal import test_util from google.protobuf.internal import type_checkers +class UnknownFieldsTest(basetest.TestCase): + + def setUp(self): + self.descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR + self.all_fields = unittest_pb2.TestAllTypes() + test_util.SetAllFields(self.all_fields) + self.all_fields_data = self.all_fields.SerializeToString() + self.empty_message = unittest_pb2.TestEmptyMessage() + self.empty_message.ParseFromString(self.all_fields_data) + + def testSerialize(self): + data = self.empty_message.SerializeToString() + + # Don't use assertEqual because we don't want to dump raw binary data to + # stdout. + self.assertTrue(data == self.all_fields_data) + + def testSerializeProto3(self): + # Verify that proto3 doesn't preserve unknown fields. + message = unittest_proto3_arena_pb2.TestEmptyMessage() + message.ParseFromString(self.all_fields_data) + self.assertEqual(0, len(message.SerializeToString())) + + def testByteSize(self): + self.assertEqual(self.all_fields.ByteSize(), self.empty_message.ByteSize()) + + def testListFields(self): + # Make sure ListFields doesn't return unknown fields. + self.assertEqual(0, len(self.empty_message.ListFields())) + + def testSerializeMessageSetWireFormatUnknownExtension(self): + # Create a message using the message set wire format with an unknown + # message. + raw = unittest_mset_pb2.RawMessageSet() + + # Add an unknown extension. + item = raw.item.add() + item.type_id = 1545009 + message1 = unittest_mset_pb2.TestMessageSetExtension1() + message1.i = 12345 + item.message = message1.SerializeToString() + + serialized = raw.SerializeToString() + + # Parse message using the message set wire format. + proto = unittest_mset_pb2.TestMessageSet() + proto.MergeFromString(serialized) + + # Verify that the unknown extension is serialized unchanged + reserialized = proto.SerializeToString() + new_raw = unittest_mset_pb2.RawMessageSet() + new_raw.MergeFromString(reserialized) + self.assertEqual(raw, new_raw) + + # C++ implementation for proto2 does not currently take into account unknown + # fields when checking equality. + # + # TODO(haberman): fix this. + @basetest.unittest.skipIf( + api_implementation.Type() == 'cpp' and api_implementation.Version() == 2, + 'C++ implementation does not expose unknown fields to Python') + def testEquals(self): + message = unittest_pb2.TestEmptyMessage() + message.ParseFromString(self.all_fields_data) + self.assertEqual(self.empty_message, message) + + self.all_fields.ClearField('optional_string') + message.ParseFromString(self.all_fields.SerializeToString()) + self.assertNotEqual(self.empty_message, message) + + @basetest.unittest.skipIf( api_implementation.Type() == 'cpp' and api_implementation.Version() == 2, 'C++ implementation does not expose unknown fields to Python') -class UnknownFieldsTest(basetest.TestCase): +class UnknownFieldsAccessorsTest(basetest.TestCase): def setUp(self): self.descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR @@ -98,13 +170,6 @@ class UnknownFieldsTest(basetest.TestCase): value = self.GetField('optionalgroup') self.assertEqual(self.all_fields.optionalgroup, value) - def testSerialize(self): - data = self.empty_message.SerializeToString() - - # Don't use assertEqual because we don't want to dump raw binary data to - # stdout. - self.assertTrue(data == self.all_fields_data) - def testCopyFrom(self): message = unittest_pb2.TestEmptyMessage() message.CopyFrom(self.empty_message) @@ -132,51 +197,12 @@ class UnknownFieldsTest(basetest.TestCase): self.empty_message.Clear() self.assertEqual(0, len(self.empty_message._unknown_fields)) - def testByteSize(self): - self.assertEqual(self.all_fields.ByteSize(), self.empty_message.ByteSize()) - def testUnknownExtensions(self): message = unittest_pb2.TestEmptyMessageWithExtensions() message.ParseFromString(self.all_fields_data) self.assertEqual(self.empty_message._unknown_fields, message._unknown_fields) - def testListFields(self): - # Make sure ListFields doesn't return unknown fields. - self.assertEqual(0, len(self.empty_message.ListFields())) - - def testSerializeMessageSetWireFormatUnknownExtension(self): - # Create a message using the message set wire format with an unknown - # message. - raw = unittest_mset_pb2.RawMessageSet() - - # Add an unknown extension. - item = raw.item.add() - item.type_id = 1545009 - message1 = unittest_mset_pb2.TestMessageSetExtension1() - message1.i = 12345 - item.message = message1.SerializeToString() - - serialized = raw.SerializeToString() - - # Parse message using the message set wire format. - proto = unittest_mset_pb2.TestMessageSet() - proto.MergeFromString(serialized) - - # Verify that the unknown extension is serialized unchanged - reserialized = proto.SerializeToString() - new_raw = unittest_mset_pb2.RawMessageSet() - new_raw.MergeFromString(reserialized) - self.assertEqual(raw, new_raw) - - def testEquals(self): - message = unittest_pb2.TestEmptyMessage() - message.ParseFromString(self.all_fields_data) - self.assertEqual(self.empty_message, message) - - self.all_fields.ClearField('optional_string') - message.ParseFromString(self.all_fields.SerializeToString()) - self.assertNotEqual(self.empty_message, message) @basetest.unittest.skipIf( diff --git a/python/google/protobuf/message.py b/python/google/protobuf/message.py index 88ed9f4c..de2f5697 100755 --- a/python/google/protobuf/message.py +++ b/python/google/protobuf/message.py @@ -36,7 +36,6 @@ __author__ = 'robinson@google.com (Will Robinson)' - class Error(Exception): pass class DecodeError(Error): pass class EncodeError(Error): pass diff --git a/python/google/protobuf/pyext/descriptor.cc b/python/google/protobuf/pyext/descriptor.cc index 7343c0b7..6890cd04 100644 --- a/python/google/protobuf/pyext/descriptor.cc +++ b/python/google/protobuf/pyext/descriptor.cc @@ -31,10 +31,15 @@ // Author: petar@google.com (Petar Petrov) #include <Python.h> +#include <frameobject.h> #include <string> +#include <google/protobuf/io/coded_stream.h> #include <google/protobuf/descriptor.pb.h> +#include <google/protobuf/dynamic_message.h> #include <google/protobuf/pyext/descriptor.h> +#include <google/protobuf/pyext/descriptor_containers.h> +#include <google/protobuf/pyext/descriptor_pool.h> #include <google/protobuf/pyext/message.h> #include <google/protobuf/pyext/scoped_pyobject_ptr.h> @@ -42,19 +47,97 @@ #if PY_MAJOR_VERSION >= 3 #define PyString_FromStringAndSize PyUnicode_FromStringAndSize + #define PyString_Check PyUnicode_Check + #define PyString_InternFromString PyUnicode_InternFromString #define PyInt_FromLong PyLong_FromLong + #define PyInt_FromSize_t PyLong_FromSize_t #if PY_VERSION_HEX < 0x03030000 #error "Python 3.0 - 3.2 are not supported." - #else - #define PyString_AsString(ob) \ - (PyUnicode_Check(ob)? PyUnicode_AsUTF8(ob): PyBytes_AsString(ob)) #endif + #define PyString_AsStringAndSize(ob, charpp, sizep) \ + (PyUnicode_Check(ob)? \ + ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \ + PyBytes_AsStringAndSize(ob, (charpp), (sizep))) #endif namespace google { namespace protobuf { namespace python { +PyObject* PyString_FromCppString(const string& str) { + return PyString_FromStringAndSize(str.c_str(), str.size()); +} + +// Check that the calling Python code is the global scope of a _pb2.py module. +// This function is used to support the current code generated by the proto +// compiler, which creates descriptors, then update some properties. +// For example: +// message_descriptor = Descriptor( +// name='Message', +// fields = [FieldDescriptor(name='field')] +// message_descriptor.fields[0].containing_type = message_descriptor +// +// This code is still executed, but the descriptors now have no other storage +// than the (const) C++ pointer, and are immutable. +// So we let this code pass, by simply ignoring the new value. +// +// From user code, descriptors still look immutable. +// +// TODO(amauryfa): Change the proto2 compiler to remove the assignments, and +// remove this hack. +bool _CalledFromGeneratedFile(int stacklevel) { + PyThreadState *state = PyThreadState_GET(); + if (state == NULL) { + return false; + } + PyFrameObject* frame = state->frame; + if (frame == NULL) { + return false; + } + while (stacklevel-- > 0) { + frame = frame->f_back; + if (frame == NULL) { + return false; + } + } + if (frame->f_globals != frame->f_locals) { + // Not at global module scope + return false; + } + + if (frame->f_code->co_filename == NULL) { + return false; + } + char* filename; + Py_ssize_t filename_size; + if (PyString_AsStringAndSize(frame->f_code->co_filename, + &filename, &filename_size) < 0) { + // filename is not a string. + PyErr_Clear(); + return false; + } + if (filename_size < 7) { + // filename is too short. + return false; + } + if (strcmp(&filename[filename_size - 7], "_pb2.py") != 0) { + // Filename is not ending with _pb2. + return false; + } + return true; +} + +// If the calling code is not a _pb2.py file, raise AttributeError. +// To be used in attribute setters. +static int CheckCalledFromGeneratedFile(const char* attr_name) { + if (_CalledFromGeneratedFile(0)) { + return 0; + } + PyErr_Format(PyExc_AttributeError, + "attribute is not writable: %s", attr_name); + return -1; +} + #ifndef PyVarObject_HEAD_INIT #define PyVarObject_HEAD_INIT(type, size) PyObject_HEAD_INIT(type) size, @@ -64,42 +147,162 @@ namespace python { #endif -static google::protobuf::DescriptorPool* g_descriptor_pool = NULL; +// Helper functions for descriptor objects. -namespace cmessage_descriptor { +// Converts options into a Python protobuf, and cache the result. +// +// This is a bit tricky because options can contain extension fields defined in +// the same proto file. In this case the options parsed from the serialized_pb +// have unkown fields, and we need to parse them again. +// +// Always returns a new reference. +template<class DescriptorClass> +static PyObject* GetOrBuildOptions(const DescriptorClass *descriptor) { + hash_map<const void*, PyObject*>* descriptor_options = + GetDescriptorPool()->descriptor_options; + // First search in the cache. + if (descriptor_options->find(descriptor) != descriptor_options->end()) { + PyObject *value = (*descriptor_options)[descriptor]; + Py_INCREF(value); + return value; + } -static void Dealloc(CMessageDescriptor* self) { - Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self)); + // Build the Options object: get its Python class, and make a copy of the C++ + // read-only instance. + const Message& options(descriptor->options()); + const Descriptor *message_type = options.GetDescriptor(); + PyObject* message_class(cdescriptor_pool::GetMessageClass( + GetDescriptorPool(), message_type)); + if (message_class == NULL) { + PyErr_Format(PyExc_TypeError, "Could not retrieve class for Options: %s", + message_type->full_name().c_str()); + return NULL; + } + ScopedPyObjectPtr value(PyEval_CallObject(message_class, NULL)); + if (value == NULL) { + return NULL; + } + CMessage* cmsg = reinterpret_cast<CMessage*>(value.get()); + + const Reflection* reflection = options.GetReflection(); + const UnknownFieldSet& unknown_fields(reflection->GetUnknownFields(options)); + if (unknown_fields.empty()) { + cmsg->message->CopyFrom(options); + } else { + // Reparse options string! XXX call cmessage::MergeFromString + string serialized; + options.SerializeToString(&serialized); + io::CodedInputStream input( + reinterpret_cast<const uint8*>(serialized.c_str()), serialized.size()); + input.SetExtensionRegistry(GetDescriptorPool()->pool, + cmessage::GetMessageFactory()); + bool success = cmsg->message->MergePartialFromCodedStream(&input); + if (!success) { + PyErr_Format(PyExc_ValueError, "Error parsing Options message"); + return NULL; + } + } + + // Cache the result. + Py_INCREF(value); + (*GetDescriptorPool()->descriptor_options)[descriptor] = value.get(); + + return value.release(); } -static PyObject* GetFullName(CMessageDescriptor* self, void *closure) { - return PyString_FromStringAndSize( - self->descriptor->full_name().c_str(), - self->descriptor->full_name().size()); +// Copy the C++ descriptor to a Python message. +// The Python message is an instance of descriptor_pb2.DescriptorProto +// or similar. +template<class DescriptorProtoClass, class DescriptorClass> +static PyObject* CopyToPythonProto(const DescriptorClass *descriptor, + PyObject *target) { + const Descriptor* self_descriptor = + DescriptorProtoClass::default_instance().GetDescriptor(); + CMessage* message = reinterpret_cast<CMessage*>(target); + if (!PyObject_TypeCheck(target, &CMessage_Type) || + message->message->GetDescriptor() != self_descriptor) { + PyErr_Format(PyExc_TypeError, "Not a %s message", + self_descriptor->full_name().c_str()); + return NULL; + } + cmessage::AssureWritable(message); + DescriptorProtoClass* descriptor_message = + static_cast<DescriptorProtoClass*>(message->message); + descriptor->CopyTo(descriptor_message); + Py_RETURN_NONE; +} + +// All Descriptors classes share the same memory layout. +typedef struct PyBaseDescriptor { + PyObject_HEAD + + // Pointer to the C++ proto2 descriptor. + // Like all descriptors, it is owned by the global DescriptorPool. + const void* descriptor; +} PyBaseDescriptor; + + +// FileDescriptor structure "inherits" from the base descriptor. +typedef struct PyFileDescriptor { + PyBaseDescriptor base; + + // The cached version of serialized pb. Either NULL, or a Bytes string. + // We own the reference. + PyObject *serialized_pb; +} PyFileDescriptor; + + +namespace descriptor { + +// Creates or retrieve a Python descriptor of the specified type. +// Objects are interned: the same descriptor will return the same object if it +// was kept alive. +// Always return a new reference. +PyObject* NewInternedDescriptor(PyTypeObject* type, const void* descriptor) { + if (descriptor == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + + // See if the object is in the map of interned descriptors + hash_map<const void*, PyObject*>::iterator it = + GetDescriptorPool()->interned_descriptors->find(descriptor); + if (it != GetDescriptorPool()->interned_descriptors->end()) { + GOOGLE_DCHECK(Py_TYPE(it->second) == type); + Py_INCREF(it->second); + return it->second; + } + // Create a new descriptor object + PyBaseDescriptor* py_descriptor = PyObject_New( + PyBaseDescriptor, type); + if (py_descriptor == NULL) { + return NULL; + } + py_descriptor->descriptor = descriptor; + // and cache it. + GetDescriptorPool()->interned_descriptors->insert( + std::make_pair(descriptor, reinterpret_cast<PyObject*>(py_descriptor))); + + return reinterpret_cast<PyObject*>(py_descriptor); } -static PyObject* GetName(CMessageDescriptor *self, void *closure) { - return PyString_FromStringAndSize( - self->descriptor->name().c_str(), - self->descriptor->name().size()); +static void Dealloc(PyBaseDescriptor* self) { + // Remove from interned dictionary + GetDescriptorPool()->interned_descriptors->erase(self->descriptor); + Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self)); } static PyGetSetDef Getters[] = { - { C("full_name"), (getter)GetFullName, NULL, "Full name", NULL}, - { C("name"), (getter)GetName, NULL, "Unqualified name", NULL}, {NULL} }; -} // namespace cmessage_descriptor - -PyTypeObject CMessageDescriptor_Type = { +PyTypeObject PyBaseDescriptor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - C("google.protobuf.internal." - "_net_proto2___python." - "CMessageDescriptor"), // tp_name - sizeof(CMessageDescriptor), // tp_basicsize + "google.protobuf.internal." + "_message.DescriptorBase", // tp_name + sizeof(PyBaseDescriptor), // tp_basicsize 0, // tp_itemsize - (destructor)cmessage_descriptor::Dealloc, // tp_dealloc + (destructor)Dealloc, // tp_dealloc 0, // tp_print 0, // tp_getattr 0, // tp_setattr @@ -115,7 +318,7 @@ PyTypeObject CMessageDescriptor_Type = { 0, // tp_setattro 0, // tp_as_buffer Py_TPFLAGS_DEFAULT, // tp_flags - C("A Message Descriptor"), // tp_doc + "Descriptors base class", // tp_doc 0, // tp_traverse 0, // tp_clear 0, // tp_richcompare @@ -124,68 +327,235 @@ PyTypeObject CMessageDescriptor_Type = { 0, // tp_iternext 0, // tp_methods 0, // tp_members - cmessage_descriptor::Getters, // tp_getset - 0, // tp_base - 0, // tp_dict - 0, // tp_descr_get - 0, // tp_descr_set - 0, // tp_dictoffset - 0, // tp_init - PyType_GenericAlloc, // tp_alloc - PyType_GenericNew, // tp_new - PyObject_Del, // tp_free + Getters, // tp_getset }; +} // namespace descriptor + +const void* PyDescriptor_AsVoidPtr(PyObject* obj) { + if (!PyObject_TypeCheck(obj, &descriptor::PyBaseDescriptor_Type)) { + PyErr_SetString(PyExc_TypeError, "Not a BaseDescriptor"); + return NULL; + } + return reinterpret_cast<PyBaseDescriptor*>(obj)->descriptor; +} -namespace cfield_descriptor { +namespace message_descriptor { -static void Dealloc(CFieldDescriptor* self) { - Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self)); +// Unchecked accessor to the C++ pointer. +static const Descriptor* _GetDescriptor(PyBaseDescriptor* self) { + return reinterpret_cast<const Descriptor*>(self->descriptor); } -static PyObject* GetFullName(CFieldDescriptor* self, void *closure) { - return PyString_FromStringAndSize( - self->descriptor->full_name().c_str(), - self->descriptor->full_name().size()); +static PyObject* GetName(PyBaseDescriptor* self, void *closure) { + return PyString_FromCppString(_GetDescriptor(self)->name()); } -static PyObject* GetName(CFieldDescriptor *self, void *closure) { - return PyString_FromStringAndSize( - self->descriptor->name().c_str(), - self->descriptor->name().size()); +static PyObject* GetFullName(PyBaseDescriptor* self, void *closure) { + return PyString_FromCppString(_GetDescriptor(self)->full_name()); } -static PyObject* GetCppType(CFieldDescriptor *self, void *closure) { - return PyInt_FromLong(self->descriptor->cpp_type()); +static PyObject* GetFile(PyBaseDescriptor *self, void *closure) { + return PyFileDescriptor_New(_GetDescriptor(self)->file()); } -static PyObject* GetLabel(CFieldDescriptor *self, void *closure) { - return PyInt_FromLong(self->descriptor->label()); +static PyObject* GetConcreteClass(PyBaseDescriptor* self, void *closure) { + PyObject* concrete_class(cdescriptor_pool::GetMessageClass( + GetDescriptorPool(), _GetDescriptor(self))); + Py_XINCREF(concrete_class); + return concrete_class; } -static PyObject* GetID(CFieldDescriptor *self, void *closure) { - return PyLong_FromVoidPtr(self); +static int SetConcreteClass(PyBaseDescriptor *self, PyObject *value, + void *closure) { + // This attribute is also set from reflection.py. Check that it's actually a + // no-op. + if (value != cdescriptor_pool::GetMessageClass( + GetDescriptorPool(), _GetDescriptor(self))) { + PyErr_SetString(PyExc_AttributeError, "Cannot change _concrete_class"); + } + return 0; +} + +static PyObject* GetFieldsByName(PyBaseDescriptor* self, void *closure) { + return NewMessageFieldsByName(_GetDescriptor(self)); +} + +static PyObject* GetFieldsByNumber(PyBaseDescriptor* self, void *closure) { + return NewMessageFieldsByNumber(_GetDescriptor(self)); +} + +static PyObject* GetFieldsSeq(PyBaseDescriptor* self, void *closure) { + return NewMessageFieldsSeq(_GetDescriptor(self)); +} + +static PyObject* GetNestedTypesByName(PyBaseDescriptor* self, void *closure) { + return NewMessageNestedTypesByName(_GetDescriptor(self)); +} + +static PyObject* GetNestedTypesSeq(PyBaseDescriptor* self, void *closure) { + return NewMessageNestedTypesSeq(_GetDescriptor(self)); +} + +static PyObject* GetExtensionsByName(PyBaseDescriptor* self, void *closure) { + return NewMessageExtensionsByName(_GetDescriptor(self)); +} + +static PyObject* GetExtensions(PyBaseDescriptor* self, void *closure) { + return NewMessageExtensionsSeq(_GetDescriptor(self)); +} + +static PyObject* GetEnumsSeq(PyBaseDescriptor* self, void *closure) { + return NewMessageEnumsSeq(_GetDescriptor(self)); +} + +static PyObject* GetEnumTypesByName(PyBaseDescriptor* self, void *closure) { + return NewMessageEnumsByName(_GetDescriptor(self)); +} + +static PyObject* GetEnumValuesByName(PyBaseDescriptor* self, void *closure) { + return NewMessageEnumValuesByName(_GetDescriptor(self)); +} + +static PyObject* GetOneofsByName(PyBaseDescriptor* self, void *closure) { + return NewMessageOneofsByName(_GetDescriptor(self)); +} + +static PyObject* GetOneofsSeq(PyBaseDescriptor* self, void *closure) { + return NewMessageOneofsSeq(_GetDescriptor(self)); +} + +static PyObject* IsExtendable(PyBaseDescriptor *self, void *closure) { + if (_GetDescriptor(self)->extension_range_count() > 0) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } +} + +static PyObject* GetExtensionRanges(PyBaseDescriptor *self, void *closure) { + const Descriptor* descriptor = _GetDescriptor(self); + PyObject* range_list = PyList_New(descriptor->extension_range_count()); + + for (int i = 0; i < descriptor->extension_range_count(); i++) { + const Descriptor::ExtensionRange* range = descriptor->extension_range(i); + PyObject* start = PyInt_FromLong(range->start); + PyObject* end = PyInt_FromLong(range->end); + PyList_SetItem(range_list, i, PyTuple_Pack(2, start, end)); + } + + return range_list; +} + +static PyObject* GetContainingType(PyBaseDescriptor *self, void *closure) { + const Descriptor* containing_type = + _GetDescriptor(self)->containing_type(); + if (containing_type) { + return PyMessageDescriptor_New(containing_type); + } else { + Py_RETURN_NONE; + } +} + +static int SetContainingType(PyBaseDescriptor *self, PyObject *value, + void *closure) { + return CheckCalledFromGeneratedFile("containing_type"); +} + +static PyObject* GetHasOptions(PyBaseDescriptor *self, void *closure) { + const MessageOptions& options(_GetDescriptor(self)->options()); + if (&options != &MessageOptions::default_instance()) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } +} +static int SetHasOptions(PyBaseDescriptor *self, PyObject *value, + void *closure) { + return CheckCalledFromGeneratedFile("has_options"); +} + +static PyObject* GetOptions(PyBaseDescriptor *self) { + return GetOrBuildOptions(_GetDescriptor(self)); +} + +static int SetOptions(PyBaseDescriptor *self, PyObject *value, + void *closure) { + return CheckCalledFromGeneratedFile("_options"); +} + +static PyObject* CopyToProto(PyBaseDescriptor *self, PyObject *target) { + return CopyToPythonProto<DescriptorProto>(_GetDescriptor(self), target); +} + +static PyObject* EnumValueName(PyBaseDescriptor *self, PyObject *args) { + const char *enum_name; + int number; + if (!PyArg_ParseTuple(args, "si", &enum_name, &number)) + return NULL; + const EnumDescriptor *enum_type = + _GetDescriptor(self)->FindEnumTypeByName(enum_name); + if (enum_type == NULL) { + PyErr_SetString(PyExc_KeyError, enum_name); + return NULL; + } + const EnumValueDescriptor *enum_value = + enum_type->FindValueByNumber(number); + if (enum_value == NULL) { + PyErr_Format(PyExc_KeyError, "%d", number); + return NULL; + } + return PyString_FromCppString(enum_value->name()); +} + +static PyObject* GetSyntax(PyBaseDescriptor *self, void *closure) { + return PyString_InternFromString( + FileDescriptor::SyntaxName(_GetDescriptor(self)->file()->syntax())); } static PyGetSetDef Getters[] = { + { C("name"), (getter)GetName, NULL, "Last name", NULL}, { C("full_name"), (getter)GetFullName, NULL, "Full name", NULL}, - { C("name"), (getter)GetName, NULL, "Unqualified name", NULL}, - { C("cpp_type"), (getter)GetCppType, NULL, "C++ Type", NULL}, - { C("label"), (getter)GetLabel, NULL, "Label", NULL}, - { C("id"), (getter)GetID, NULL, "ID", NULL}, + { C("_concrete_class"), (getter)GetConcreteClass, (setter)SetConcreteClass, "concrete class", NULL}, + { C("file"), (getter)GetFile, NULL, "File descriptor", NULL}, + + { C("fields"), (getter)GetFieldsSeq, NULL, "Fields sequence", NULL}, + { C("fields_by_name"), (getter)GetFieldsByName, NULL, "Fields by name", NULL}, + { C("fields_by_number"), (getter)GetFieldsByNumber, NULL, "Fields by number", NULL}, + { C("nested_types"), (getter)GetNestedTypesSeq, NULL, "Nested types sequence", NULL}, + { C("nested_types_by_name"), (getter)GetNestedTypesByName, NULL, "Nested types by name", NULL}, + { C("extensions"), (getter)GetExtensions, NULL, "Extensions Sequence", NULL}, + { C("extensions_by_name"), (getter)GetExtensionsByName, NULL, "Extensions by name", NULL}, + { C("extension_ranges"), (getter)GetExtensionRanges, NULL, "Extension ranges", NULL}, + { C("enum_types"), (getter)GetEnumsSeq, NULL, "Enum sequence", NULL}, + { C("enum_types_by_name"), (getter)GetEnumTypesByName, NULL, "Enum types by name", NULL}, + { C("enum_values_by_name"), (getter)GetEnumValuesByName, NULL, "Enum values by name", NULL}, + { C("oneofs_by_name"), (getter)GetOneofsByName, NULL, "Oneofs by name", NULL}, + { C("oneofs"), (getter)GetOneofsSeq, NULL, "Oneofs by name", NULL}, + { C("containing_type"), (getter)GetContainingType, (setter)SetContainingType, "Containing type", NULL}, + { C("is_extendable"), (getter)IsExtendable, (setter)NULL, NULL, NULL}, + { C("has_options"), (getter)GetHasOptions, (setter)SetHasOptions, "Has Options", NULL}, + { C("_options"), (getter)NULL, (setter)SetOptions, "Options", NULL}, + { C("syntax"), (getter)GetSyntax, (setter)NULL, "Syntax", NULL}, + {NULL} +}; + +static PyMethodDef Methods[] = { + { "GetOptions", (PyCFunction)GetOptions, METH_NOARGS, }, + { "CopyToProto", (PyCFunction)CopyToProto, METH_O, }, + { "EnumValueName", (PyCFunction)EnumValueName, METH_VARARGS, }, {NULL} }; -} // namespace cfield_descriptor +} // namespace message_descriptor -PyTypeObject CFieldDescriptor_Type = { +PyTypeObject PyMessageDescriptor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) C("google.protobuf.internal." - "_net_proto2___python." - "CFieldDescriptor"), // tp_name - sizeof(CFieldDescriptor), // tp_basicsize + "_message.MessageDescriptor"), // tp_name + sizeof(PyBaseDescriptor), // tp_basicsize 0, // tp_itemsize - (destructor)cfield_descriptor::Dealloc, // tp_dealloc + 0, // tp_dealloc 0, // tp_print 0, // tp_getattr 0, // tp_setattr @@ -201,304 +571,893 @@ PyTypeObject CFieldDescriptor_Type = { 0, // tp_setattro 0, // tp_as_buffer Py_TPFLAGS_DEFAULT, // tp_flags - C("A Field Descriptor"), // tp_doc + C("A Message Descriptor"), // tp_doc 0, // tp_traverse 0, // tp_clear 0, // tp_richcompare 0, // tp_weaklistoffset 0, // tp_iter 0, // tp_iternext - 0, // tp_methods + message_descriptor::Methods, // tp_methods 0, // tp_members - cfield_descriptor::Getters, // tp_getset - 0, // tp_base - 0, // tp_dict - 0, // tp_descr_get - 0, // tp_descr_set - 0, // tp_dictoffset - 0, // tp_init - PyType_GenericAlloc, // tp_alloc - PyType_GenericNew, // tp_new - PyObject_Del, // tp_free + message_descriptor::Getters, // tp_getset + &descriptor::PyBaseDescriptor_Type, // tp_base }; +PyObject* PyMessageDescriptor_New( + const Descriptor* message_descriptor) { + return descriptor::NewInternedDescriptor( + &PyMessageDescriptor_Type, message_descriptor); +} -namespace cdescriptor_pool { - -PyDescriptorPool* NewDescriptorPool() { - PyDescriptorPool* cdescriptor_pool = PyObject_New( - PyDescriptorPool, &PyDescriptorPool_Type); - if (cdescriptor_pool == NULL) { +const Descriptor* PyMessageDescriptor_AsDescriptor(PyObject* obj) { + if (!PyObject_TypeCheck(obj, &PyMessageDescriptor_Type)) { + PyErr_SetString(PyExc_TypeError, "Not a MessageDescriptor"); return NULL; } + return reinterpret_cast<const Descriptor*>( + reinterpret_cast<PyBaseDescriptor*>(obj)->descriptor); +} - // Build a DescriptorPool for messages only declared in Python libraries. - // generated_pool() contains all messages linked in C++ libraries, and is used - // as underlay. - cdescriptor_pool->pool = new google::protobuf::DescriptorPool( - google::protobuf::DescriptorPool::generated_pool()); +namespace field_descriptor { - // TODO(amauryfa): Rewrite the SymbolDatabase in C so that it uses the same - // storage. - cdescriptor_pool->classes_by_descriptor = - new PyDescriptorPool::ClassesByMessageMap(); +// Unchecked accessor to the C++ pointer. +static const FieldDescriptor* _GetDescriptor( + PyBaseDescriptor *self) { + return reinterpret_cast<const FieldDescriptor*>(self->descriptor); +} - return cdescriptor_pool; +static PyObject* GetFullName(PyBaseDescriptor* self, void *closure) { + return PyString_FromCppString(_GetDescriptor(self)->full_name()); } -static void Dealloc(PyDescriptorPool* self) { - typedef PyDescriptorPool::ClassesByMessageMap::iterator iterator; - for (iterator it = self->classes_by_descriptor->begin(); - it != self->classes_by_descriptor->end(); ++it) { - Py_DECREF(it->second); - } - delete self->classes_by_descriptor; - Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self)); +static PyObject* GetName(PyBaseDescriptor *self, void *closure) { + return PyString_FromCppString(_GetDescriptor(self)->name()); } -const google::protobuf::Descriptor* FindMessageTypeByName(PyDescriptorPool* self, - const string& name) { - return self->pool->FindMessageTypeByName(name); +static PyObject* GetType(PyBaseDescriptor *self, void *closure) { + return PyInt_FromLong(_GetDescriptor(self)->type()); } -static PyObject* NewCMessageDescriptor( - const google::protobuf::Descriptor* message_descriptor) { - CMessageDescriptor* cmessage_descriptor = PyObject_New( - CMessageDescriptor, &CMessageDescriptor_Type); - if (cmessage_descriptor == NULL) { - return NULL; - } - cmessage_descriptor->descriptor = message_descriptor; +static PyObject* GetCppType(PyBaseDescriptor *self, void *closure) { + return PyInt_FromLong(_GetDescriptor(self)->cpp_type()); +} - return reinterpret_cast<PyObject*>(cmessage_descriptor); +static PyObject* GetLabel(PyBaseDescriptor *self, void *closure) { + return PyInt_FromLong(_GetDescriptor(self)->label()); } -static PyObject* NewCFieldDescriptor( - const google::protobuf::FieldDescriptor* field_descriptor) { - CFieldDescriptor* cfield_descriptor = PyObject_New( - CFieldDescriptor, &CFieldDescriptor_Type); - if (cfield_descriptor == NULL) { - return NULL; - } - cfield_descriptor->descriptor = field_descriptor; +static PyObject* GetNumber(PyBaseDescriptor *self, void *closure) { + return PyInt_FromLong(_GetDescriptor(self)->number()); +} - return reinterpret_cast<PyObject*>(cfield_descriptor); +static PyObject* GetIndex(PyBaseDescriptor *self, void *closure) { + return PyInt_FromLong(_GetDescriptor(self)->index()); } -// Add a message class to our database. -const google::protobuf::Descriptor* RegisterMessageClass( - PyDescriptorPool* self, PyObject *message_class, PyObject* descriptor) { - ScopedPyObjectPtr full_message_name( - PyObject_GetAttrString(descriptor, "full_name")); - const char* full_name = PyString_AsString(full_message_name); - if (full_name == NULL) { - return NULL; +static PyObject* GetID(PyBaseDescriptor *self, void *closure) { + return PyLong_FromVoidPtr(self); +} + +static PyObject* IsExtension(PyBaseDescriptor *self, void *closure) { + return PyBool_FromLong(_GetDescriptor(self)->is_extension()); +} + +static PyObject* HasDefaultValue(PyBaseDescriptor *self, void *closure) { + return PyBool_FromLong(_GetDescriptor(self)->has_default_value()); +} + +static PyObject* GetDefaultValue(PyBaseDescriptor *self, void *closure) { + PyObject *result; + + switch (_GetDescriptor(self)->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: { + int32 value = _GetDescriptor(self)->default_value_int32(); + result = PyInt_FromLong(value); + break; + } + case FieldDescriptor::CPPTYPE_INT64: { + int64 value = _GetDescriptor(self)->default_value_int64(); + result = PyLong_FromLongLong(value); + break; + } + case FieldDescriptor::CPPTYPE_UINT32: { + uint32 value = _GetDescriptor(self)->default_value_uint32(); + result = PyInt_FromSize_t(value); + break; + } + case FieldDescriptor::CPPTYPE_UINT64: { + uint64 value = _GetDescriptor(self)->default_value_uint64(); + result = PyLong_FromUnsignedLongLong(value); + break; + } + case FieldDescriptor::CPPTYPE_FLOAT: { + float value = _GetDescriptor(self)->default_value_float(); + result = PyFloat_FromDouble(value); + break; + } + case FieldDescriptor::CPPTYPE_DOUBLE: { + double value = _GetDescriptor(self)->default_value_double(); + result = PyFloat_FromDouble(value); + break; + } + case FieldDescriptor::CPPTYPE_BOOL: { + bool value = _GetDescriptor(self)->default_value_bool(); + result = PyBool_FromLong(value); + break; + } + case FieldDescriptor::CPPTYPE_STRING: { + string value = _GetDescriptor(self)->default_value_string(); + result = ToStringObject(_GetDescriptor(self), value); + break; + } + case FieldDescriptor::CPPTYPE_ENUM: { + const EnumValueDescriptor* value = + _GetDescriptor(self)->default_value_enum(); + result = PyInt_FromLong(value->number()); + break; + } + default: + PyErr_Format(PyExc_NotImplementedError, "default value for %s", + _GetDescriptor(self)->full_name().c_str()); + return NULL; } - const Descriptor *message_descriptor = - self->pool->FindMessageTypeByName(full_name); - if (!message_descriptor) { - PyErr_Format(PyExc_TypeError, "Could not find C++ descriptor for '%s'", - full_name); - return NULL; + return result; +} + +static PyObject* GetCDescriptor(PyObject *self, void *closure) { + Py_INCREF(self); + return self; +} + +static PyObject *GetEnumType(PyBaseDescriptor *self, void *closure) { + const EnumDescriptor* enum_type = _GetDescriptor(self)->enum_type(); + if (enum_type) { + return PyEnumDescriptor_New(enum_type); + } else { + Py_RETURN_NONE; } - Py_INCREF(message_class); - typedef PyDescriptorPool::ClassesByMessageMap::iterator iterator; - std::pair<iterator, bool> ret = self->classes_by_descriptor->insert( - make_pair(message_descriptor, message_class)); - if (!ret.second) { - // Update case: DECREF the previous value. - Py_DECREF(ret.first->second); - ret.first->second = message_class; +} + +static int SetEnumType(PyBaseDescriptor *self, PyObject *value, void *closure) { + return CheckCalledFromGeneratedFile("enum_type"); +} + +static PyObject *GetMessageType(PyBaseDescriptor *self, void *closure) { + const Descriptor* message_type = _GetDescriptor(self)->message_type(); + if (message_type) { + return PyMessageDescriptor_New(message_type); + } else { + Py_RETURN_NONE; } +} - // Also add the C++ descriptor to the Python descriptor class. - ScopedPyObjectPtr cdescriptor(NewCMessageDescriptor(message_descriptor)); - if (cdescriptor == NULL) { - return NULL; +static int SetMessageType(PyBaseDescriptor *self, PyObject *value, + void *closure) { + return CheckCalledFromGeneratedFile("message_type"); +} + +static PyObject* GetContainingType(PyBaseDescriptor *self, void *closure) { + const Descriptor* containing_type = + _GetDescriptor(self)->containing_type(); + if (containing_type) { + return PyMessageDescriptor_New(containing_type); + } else { + Py_RETURN_NONE; } - if (PyObject_SetAttrString( - descriptor, "_cdescriptor", cdescriptor) < 0) { - return NULL; +} + +static int SetContainingType(PyBaseDescriptor *self, PyObject *value, + void *closure) { + return CheckCalledFromGeneratedFile("containing_type"); +} + +static PyObject* GetExtensionScope(PyBaseDescriptor *self, void *closure) { + const Descriptor* extension_scope = + _GetDescriptor(self)->extension_scope(); + if (extension_scope) { + return PyMessageDescriptor_New(extension_scope); + } else { + Py_RETURN_NONE; } - return message_descriptor; } -// Retrieve the message class added to our database. -PyObject *GetMessageClass(PyDescriptorPool* self, - const Descriptor *message_descriptor) { - typedef PyDescriptorPool::ClassesByMessageMap::iterator iterator; - iterator ret = self->classes_by_descriptor->find(message_descriptor); - if (ret == self->classes_by_descriptor->end()) { - PyErr_Format(PyExc_TypeError, "No message class registered for '%s'", - message_descriptor->full_name().c_str()); - return NULL; +static PyObject* GetContainingOneof(PyBaseDescriptor *self, void *closure) { + const OneofDescriptor* containing_oneof = + _GetDescriptor(self)->containing_oneof(); + if (containing_oneof) { + return PyOneofDescriptor_New(containing_oneof); } else { - return ret->second; + Py_RETURN_NONE; } } -PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* name) { - const char* full_field_name = PyString_AsString(name); - if (full_field_name == NULL) { - return NULL; +static int SetContainingOneof(PyBaseDescriptor *self, PyObject *value, + void *closure) { + return CheckCalledFromGeneratedFile("containing_oneof"); +} + +static PyObject* GetHasOptions(PyBaseDescriptor *self, void *closure) { + const FieldOptions& options(_GetDescriptor(self)->options()); + if (&options != &FieldOptions::default_instance()) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; } +} +static int SetHasOptions(PyBaseDescriptor *self, PyObject *value, + void *closure) { + return CheckCalledFromGeneratedFile("has_options"); +} + +static PyObject* GetOptions(PyBaseDescriptor *self) { + return GetOrBuildOptions(_GetDescriptor(self)); +} + +static int SetOptions(PyBaseDescriptor *self, PyObject *value, + void *closure) { + return CheckCalledFromGeneratedFile("_options"); +} + + +static PyGetSetDef Getters[] = { + { C("full_name"), (getter)GetFullName, NULL, "Full name", NULL}, + { C("name"), (getter)GetName, NULL, "Unqualified name", NULL}, + { C("type"), (getter)GetType, NULL, "C++ Type", NULL}, + { C("cpp_type"), (getter)GetCppType, NULL, "C++ Type", NULL}, + { C("label"), (getter)GetLabel, NULL, "Label", NULL}, + { C("number"), (getter)GetNumber, NULL, "Number", NULL}, + { C("index"), (getter)GetIndex, NULL, "Index", NULL}, + { C("default_value"), (getter)GetDefaultValue, NULL, "Default Value", NULL}, + { C("has_default_value"), (getter)HasDefaultValue, NULL, NULL, NULL}, + { C("is_extension"), (getter)IsExtension, NULL, "ID", NULL}, + { C("id"), (getter)GetID, NULL, "ID", NULL}, + { C("_cdescriptor"), (getter)GetCDescriptor, NULL, "HAACK REMOVE ME", NULL}, + + { C("message_type"), (getter)GetMessageType, (setter)SetMessageType, "Message type", NULL}, + { C("enum_type"), (getter)GetEnumType, (setter)SetEnumType, "Enum type", NULL}, + { C("containing_type"), (getter)GetContainingType, (setter)SetContainingType, "Containing type", NULL}, + { C("extension_scope"), (getter)GetExtensionScope, (setter)NULL, "Extension scope", NULL}, + { C("containing_oneof"), (getter)GetContainingOneof, (setter)SetContainingOneof, "Containing oneof", NULL}, + { C("has_options"), (getter)GetHasOptions, (setter)SetHasOptions, "Has Options", NULL}, + { C("_options"), (getter)NULL, (setter)SetOptions, "Options", NULL}, + {NULL} +}; + +static PyMethodDef Methods[] = { + { "GetOptions", (PyCFunction)GetOptions, METH_NOARGS, }, + {NULL} +}; - const google::protobuf::FieldDescriptor* field_descriptor = NULL; +} // namespace field_descriptor + +PyTypeObject PyFieldDescriptor_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + C("google.protobuf.internal." + "_message.FieldDescriptor"), // tp_name + sizeof(PyBaseDescriptor), // tp_basicsize + 0, // tp_itemsize + 0, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + C("A Field Descriptor"), // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + field_descriptor::Methods, // tp_methods + 0, // tp_members + field_descriptor::Getters, // tp_getset + &descriptor::PyBaseDescriptor_Type, // tp_base +}; - field_descriptor = self->pool->FindFieldByName(full_field_name); +PyObject* PyFieldDescriptor_New( + const FieldDescriptor* field_descriptor) { + return descriptor::NewInternedDescriptor( + &PyFieldDescriptor_Type, field_descriptor); +} - if (field_descriptor == NULL) { - PyErr_Format(PyExc_TypeError, "Couldn't find field %.200s", - full_field_name); +const FieldDescriptor* PyFieldDescriptor_AsDescriptor(PyObject* obj) { + if (!PyObject_TypeCheck(obj, &PyFieldDescriptor_Type)) { + PyErr_SetString(PyExc_TypeError, "Not a FieldDescriptor"); return NULL; } + return reinterpret_cast<const FieldDescriptor*>( + reinterpret_cast<PyBaseDescriptor*>(obj)->descriptor); +} + +namespace enum_descriptor { - return NewCFieldDescriptor(field_descriptor); +// Unchecked accessor to the C++ pointer. +static const EnumDescriptor* _GetDescriptor( + PyBaseDescriptor *self) { + return reinterpret_cast<const EnumDescriptor*>(self->descriptor); } -PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg) { - const char* full_field_name = PyString_AsString(arg); - if (full_field_name == NULL) { - return NULL; +static PyObject* GetFullName(PyBaseDescriptor* self, void *closure) { + return PyString_FromCppString(_GetDescriptor(self)->full_name()); +} + +static PyObject* GetName(PyBaseDescriptor *self, void *closure) { + return PyString_FromCppString(_GetDescriptor(self)->name()); +} + +static PyObject* GetFile(PyBaseDescriptor *self, void *closure) { + return PyFileDescriptor_New(_GetDescriptor(self)->file()); +} + +static PyObject* GetEnumvaluesByName(PyBaseDescriptor* self, void *closure) { + return NewEnumValuesByName(_GetDescriptor(self)); +} + +static PyObject* GetEnumvaluesByNumber(PyBaseDescriptor* self, void *closure) { + return NewEnumValuesByNumber(_GetDescriptor(self)); +} + +static PyObject* GetEnumvaluesSeq(PyBaseDescriptor* self, void *closure) { + return NewEnumValuesSeq(_GetDescriptor(self)); +} + +static PyObject* GetContainingType(PyBaseDescriptor *self, void *closure) { + const Descriptor* containing_type = + _GetDescriptor(self)->containing_type(); + if (containing_type) { + return PyMessageDescriptor_New(containing_type); + } else { + Py_RETURN_NONE; } +} - const google::protobuf::FieldDescriptor* field_descriptor = - self->pool->FindExtensionByName(full_field_name); - if (field_descriptor == NULL) { - PyErr_Format(PyExc_TypeError, "Couldn't find field %.200s", - full_field_name); - return NULL; +static int SetContainingType(PyBaseDescriptor *self, PyObject *value, + void *closure) { + return CheckCalledFromGeneratedFile("containing_type"); +} + + +static PyObject* GetHasOptions(PyBaseDescriptor *self, void *closure) { + const EnumOptions& options(_GetDescriptor(self)->options()); + if (&options != &EnumOptions::default_instance()) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; } +} +static int SetHasOptions(PyBaseDescriptor *self, PyObject *value, + void *closure) { + return CheckCalledFromGeneratedFile("has_options"); +} + +static PyObject* GetOptions(PyBaseDescriptor *self) { + return GetOrBuildOptions(_GetDescriptor(self)); +} - return NewCFieldDescriptor(field_descriptor); +static int SetOptions(PyBaseDescriptor *self, PyObject *value, + void *closure) { + return CheckCalledFromGeneratedFile("_options"); +} + +static PyObject* CopyToProto(PyBaseDescriptor *self, PyObject *target) { + return CopyToPythonProto<EnumDescriptorProto>(_GetDescriptor(self), target); } static PyMethodDef Methods[] = { - { C("FindFieldByName"), - (PyCFunction)FindFieldByName, - METH_O, - C("Searches for a field descriptor by full name.") }, - { C("FindExtensionByName"), - (PyCFunction)FindExtensionByName, - METH_O, - C("Searches for extension descriptor by full name.") }, + { "GetOptions", (PyCFunction)GetOptions, METH_NOARGS, }, + { "CopyToProto", (PyCFunction)CopyToProto, METH_O, }, + {NULL} +}; + +static PyGetSetDef Getters[] = { + { C("full_name"), (getter)GetFullName, NULL, "Full name", NULL}, + { C("name"), (getter)GetName, NULL, "last name", NULL}, + { C("file"), (getter)GetFile, NULL, "File descriptor", NULL}, + { C("values"), (getter)GetEnumvaluesSeq, NULL, "values", NULL}, + { C("values_by_name"), (getter)GetEnumvaluesByName, NULL, "Enumvalues by name", NULL}, + { C("values_by_number"), (getter)GetEnumvaluesByNumber, NULL, "Enumvalues by number", NULL}, + + { C("containing_type"), (getter)GetContainingType, (setter)SetContainingType, "Containing type", NULL}, + { C("has_options"), (getter)GetHasOptions, (setter)SetHasOptions, "Has Options", NULL}, + { C("_options"), (getter)NULL, (setter)SetOptions, "Options", NULL}, {NULL} }; -} // namespace cdescriptor_pool +} // namespace enum_descriptor -PyTypeObject PyDescriptorPool_Type = { +PyTypeObject PyEnumDescriptor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) C("google.protobuf.internal." - "_net_proto2___python." - "CFieldDescriptor"), // tp_name - sizeof(PyDescriptorPool), // tp_basicsize - 0, // tp_itemsize - (destructor)cdescriptor_pool::Dealloc, // tp_dealloc - 0, // tp_print - 0, // tp_getattr - 0, // tp_setattr - 0, // tp_compare - 0, // tp_repr - 0, // tp_as_number - 0, // tp_as_sequence - 0, // tp_as_mapping - 0, // tp_hash - 0, // tp_call - 0, // tp_str - 0, // tp_getattro - 0, // tp_setattro - 0, // tp_as_buffer - Py_TPFLAGS_DEFAULT, // tp_flags - C("A Descriptor Pool"), // tp_doc - 0, // tp_traverse - 0, // tp_clear - 0, // tp_richcompare - 0, // tp_weaklistoffset - 0, // tp_iter - 0, // tp_iternext - cdescriptor_pool::Methods, // tp_methods - 0, // tp_members - 0, // tp_getset - 0, // tp_base - 0, // tp_dict - 0, // tp_descr_get - 0, // tp_descr_set - 0, // tp_dictoffset - 0, // tp_init - 0, // tp_alloc - 0, // tp_new - PyObject_Del, // tp_free + "_message.EnumDescriptor"), // tp_name + sizeof(PyBaseDescriptor), // tp_basicsize + 0, // tp_itemsize + 0, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + C("A Enum Descriptor"), // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + enum_descriptor::Methods, // tp_getset + 0, // tp_members + enum_descriptor::Getters, // tp_getset + &descriptor::PyBaseDescriptor_Type, // tp_base }; +PyObject* PyEnumDescriptor_New( + const EnumDescriptor* enum_descriptor) { + return descriptor::NewInternedDescriptor( + &PyEnumDescriptor_Type, enum_descriptor); +} -// Collects errors that occur during proto file building to allow them to be -// propagated in the python exception instead of only living in ERROR logs. -class BuildFileErrorCollector : public google::protobuf::DescriptorPool::ErrorCollector { - public: - BuildFileErrorCollector() : error_message(""), had_errors(false) {} - - void AddError(const string& filename, const string& element_name, - const Message* descriptor, ErrorLocation location, - const string& message) { - // Replicates the logging behavior that happens in the C++ implementation - // when an error collector is not passed in. - if (!had_errors) { - error_message += - ("Invalid proto descriptor for file \"" + filename + "\":\n"); - } - // As this only happens on failure and will result in the program not - // running at all, no effort is made to optimize this string manipulation. - error_message += (" " + element_name + ": " + message + "\n"); +namespace enumvalue_descriptor { + +// Unchecked accessor to the C++ pointer. +static const EnumValueDescriptor* _GetDescriptor( + PyBaseDescriptor *self) { + return reinterpret_cast<const EnumValueDescriptor*>(self->descriptor); +} + +static PyObject* GetName(PyBaseDescriptor *self, void *closure) { + return PyString_FromCppString(_GetDescriptor(self)->name()); +} + +static PyObject* GetNumber(PyBaseDescriptor *self, void *closure) { + return PyInt_FromLong(_GetDescriptor(self)->number()); +} + +static PyObject* GetIndex(PyBaseDescriptor *self, void *closure) { + return PyInt_FromLong(_GetDescriptor(self)->index()); +} + +static PyObject* GetType(PyBaseDescriptor *self, void *closure) { + return PyEnumDescriptor_New(_GetDescriptor(self)->type()); +} + +static PyObject* GetHasOptions(PyBaseDescriptor *self, void *closure) { + const EnumValueOptions& options(_GetDescriptor(self)->options()); + if (&options != &EnumValueOptions::default_instance()) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; } +} +static int SetHasOptions(PyBaseDescriptor *self, PyObject *value, + void *closure) { + return CheckCalledFromGeneratedFile("has_options"); +} + +static PyObject* GetOptions(PyBaseDescriptor *self) { + return GetOrBuildOptions(_GetDescriptor(self)); +} + +static int SetOptions(PyBaseDescriptor *self, PyObject *value, + void *closure) { + return CheckCalledFromGeneratedFile("_options"); +} + + +static PyGetSetDef Getters[] = { + { C("name"), (getter)GetName, NULL, "name", NULL}, + { C("number"), (getter)GetNumber, NULL, "number", NULL}, + { C("index"), (getter)GetIndex, NULL, "index", NULL}, + { C("type"), (getter)GetType, NULL, "index", NULL}, + + { C("has_options"), (getter)GetHasOptions, (setter)SetHasOptions, "Has Options", NULL}, + { C("_options"), (getter)NULL, (setter)SetOptions, "Options", NULL}, + {NULL} +}; + +static PyMethodDef Methods[] = { + { "GetOptions", (PyCFunction)GetOptions, METH_NOARGS, }, + {NULL} +}; + +} // namespace enumvalue_descriptor - string error_message; - bool had_errors; +PyTypeObject PyEnumValueDescriptor_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + C("google.protobuf.internal." + "_message.EnumValueDescriptor"), // tp_name + sizeof(PyBaseDescriptor), // tp_basicsize + 0, // tp_itemsize + 0, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + C("A EnumValue Descriptor"), // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + enumvalue_descriptor::Methods, // tp_methods + 0, // tp_members + enumvalue_descriptor::Getters, // tp_getset + &descriptor::PyBaseDescriptor_Type, // tp_base }; -PyObject* Python_BuildFile(PyObject* ignored, PyObject* arg) { - char* message_type; - Py_ssize_t message_len; +PyObject* PyEnumValueDescriptor_New( + const EnumValueDescriptor* enumvalue_descriptor) { + return descriptor::NewInternedDescriptor( + &PyEnumValueDescriptor_Type, enumvalue_descriptor); +} + +namespace file_descriptor { + +// Unchecked accessor to the C++ pointer. +static const FileDescriptor* _GetDescriptor(PyFileDescriptor *self) { + return reinterpret_cast<const FileDescriptor*>(self->base.descriptor); +} + +static void Dealloc(PyFileDescriptor* self) { + Py_XDECREF(self->serialized_pb); + descriptor::Dealloc(&self->base); +} + +static PyObject* GetName(PyFileDescriptor *self, void *closure) { + return PyString_FromCppString(_GetDescriptor(self)->name()); +} + +static PyObject* GetPackage(PyFileDescriptor *self, void *closure) { + return PyString_FromCppString(_GetDescriptor(self)->package()); +} - if (PyBytes_AsStringAndSize(arg, &message_type, &message_len) < 0) { +static PyObject* GetSerializedPb(PyFileDescriptor *self, void *closure) { + PyObject *serialized_pb = self->serialized_pb; + if (serialized_pb != NULL) { + Py_INCREF(serialized_pb); + return serialized_pb; + } + FileDescriptorProto file_proto; + _GetDescriptor(self)->CopyTo(&file_proto); + string contents; + file_proto.SerializePartialToString(&contents); + self->serialized_pb = PyBytes_FromStringAndSize( + contents.c_str(), contents.size()); + if (self->serialized_pb == NULL) { return NULL; } + Py_INCREF(self->serialized_pb); + return self->serialized_pb; +} + +static PyObject* GetMessageTypesByName(PyFileDescriptor* self, void *closure) { + return NewFileMessageTypesByName(_GetDescriptor(self)); +} + +static PyObject* GetEnumTypesByName(PyFileDescriptor* self, void *closure) { + return NewFileEnumTypesByName(_GetDescriptor(self)); +} + +static PyObject* GetExtensionsByName(PyFileDescriptor* self, void *closure) { + return NewFileExtensionsByName(_GetDescriptor(self)); +} + +static PyObject* GetDependencies(PyFileDescriptor* self, void *closure) { + return NewFileDependencies(_GetDescriptor(self)); +} + +static PyObject* GetPublicDependencies(PyFileDescriptor* self, void *closure) { + return NewFilePublicDependencies(_GetDescriptor(self)); +} + +static PyObject* GetHasOptions(PyFileDescriptor *self, void *closure) { + const FileOptions& options(_GetDescriptor(self)->options()); + if (&options != &FileOptions::default_instance()) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } +} +static int SetHasOptions(PyFileDescriptor *self, PyObject *value, + void *closure) { + return CheckCalledFromGeneratedFile("has_options"); +} + +static PyObject* GetOptions(PyFileDescriptor *self) { + return GetOrBuildOptions(_GetDescriptor(self)); +} + +static int SetOptions(PyFileDescriptor *self, PyObject *value, + void *closure) { + return CheckCalledFromGeneratedFile("_options"); +} + +static PyObject* GetSyntax(PyFileDescriptor *self, void *closure) { + return PyString_InternFromString( + FileDescriptor::SyntaxName(_GetDescriptor(self)->syntax())); +} - google::protobuf::FileDescriptorProto file_proto; - if (!file_proto.ParseFromArray(message_type, message_len)) { - PyErr_SetString(PyExc_TypeError, "Couldn't parse file content!"); +static PyObject* CopyToProto(PyFileDescriptor *self, PyObject *target) { + return CopyToPythonProto<FileDescriptorProto>(_GetDescriptor(self), target); +} + +static PyGetSetDef Getters[] = { + { C("name"), (getter)GetName, NULL, "name", NULL}, + { C("package"), (getter)GetPackage, NULL, "package", NULL}, + { C("serialized_pb"), (getter)GetSerializedPb, NULL, NULL, NULL}, + { C("message_types_by_name"), (getter)GetMessageTypesByName, NULL, "Messages by name", NULL}, + { C("enum_types_by_name"), (getter)GetEnumTypesByName, NULL, "Enums by name", NULL}, + { C("extensions_by_name"), (getter)GetExtensionsByName, NULL, "Extensions by name", NULL}, + { C("dependencies"), (getter)GetDependencies, NULL, "Dependencies", NULL}, + { C("public_dependencies"), (getter)GetPublicDependencies, NULL, "Dependencies", NULL}, + + { C("has_options"), (getter)GetHasOptions, (setter)SetHasOptions, "Has Options", NULL}, + { C("_options"), (getter)NULL, (setter)SetOptions, "Options", NULL}, + { C("syntax"), (getter)GetSyntax, (setter)NULL, "Syntax", NULL}, + {NULL} +}; + +static PyMethodDef Methods[] = { + { "GetOptions", (PyCFunction)GetOptions, METH_NOARGS, }, + { "CopyToProto", (PyCFunction)CopyToProto, METH_O, }, + {NULL} +}; + +} // namespace file_descriptor + +PyTypeObject PyFileDescriptor_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + C("google.protobuf.internal." + "_message.FileDescriptor"), // tp_name + sizeof(PyFileDescriptor), // tp_basicsize + 0, // tp_itemsize + (destructor)file_descriptor::Dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + C("A File Descriptor"), // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + file_descriptor::Methods, // tp_methods + 0, // tp_members + file_descriptor::Getters, // tp_getset + &descriptor::PyBaseDescriptor_Type, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init + PyType_GenericAlloc, // tp_alloc + PyType_GenericNew, // tp_new + PyObject_Del, // tp_free +}; + +PyObject* PyFileDescriptor_New(const FileDescriptor* file_descriptor) { + return descriptor::NewInternedDescriptor( + &PyFileDescriptor_Type, file_descriptor); +} + +PyObject* PyFileDescriptor_NewWithPb( + const FileDescriptor* file_descriptor, PyObject *serialized_pb) { + PyObject* py_descriptor = PyFileDescriptor_New(file_descriptor); + if (py_descriptor == NULL) { return NULL; } + if (serialized_pb != NULL) { + PyFileDescriptor* cfile_descriptor = + reinterpret_cast<PyFileDescriptor*>(py_descriptor); + Py_XINCREF(serialized_pb); + cfile_descriptor->serialized_pb = serialized_pb; + } + + return py_descriptor; +} + +namespace oneof_descriptor { + +// Unchecked accessor to the C++ pointer. +static const OneofDescriptor* _GetDescriptor( + PyBaseDescriptor *self) { + return reinterpret_cast<const OneofDescriptor*>(self->descriptor); +} + +static PyObject* GetName(PyBaseDescriptor* self, void *closure) { + return PyString_FromCppString(_GetDescriptor(self)->name()); +} + +static PyObject* GetFullName(PyBaseDescriptor* self, void *closure) { + return PyString_FromCppString(_GetDescriptor(self)->full_name()); +} + +static PyObject* GetIndex(PyBaseDescriptor *self, void *closure) { + return PyInt_FromLong(_GetDescriptor(self)->index()); +} - // If the file was already part of a C++ library, all its descriptors are in - // the underlying pool. No need to do anything else. - if (google::protobuf::DescriptorPool::generated_pool()->FindFileByName( - file_proto.name()) != NULL) { +static PyObject* GetFields(PyBaseDescriptor* self, void *closure) { + return NewOneofFieldsSeq(_GetDescriptor(self)); +} + +static PyObject* GetContainingType(PyBaseDescriptor *self, void *closure) { + const Descriptor* containing_type = + _GetDescriptor(self)->containing_type(); + if (containing_type) { + return PyMessageDescriptor_New(containing_type); + } else { Py_RETURN_NONE; } +} - BuildFileErrorCollector error_collector; - const google::protobuf::FileDescriptor* descriptor = - GetDescriptorPool()->pool->BuildFileCollectingErrors(file_proto, - &error_collector); - if (descriptor == NULL) { - PyErr_Format(PyExc_TypeError, - "Couldn't build proto file into descriptor pool!\n%s", - error_collector.error_message.c_str()); - return NULL; +static PyGetSetDef Getters[] = { + { C("name"), (getter)GetName, NULL, "Name", NULL}, + { C("full_name"), (getter)GetFullName, NULL, "Full name", NULL}, + { C("index"), (getter)GetIndex, NULL, "Index", NULL}, + + { C("containing_type"), (getter)GetContainingType, NULL, "Containing type", NULL}, + { C("fields"), (getter)GetFields, NULL, "Fields", NULL}, + {NULL} +}; + +} // namespace oneof_descriptor + +PyTypeObject PyOneofDescriptor_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + C("google.protobuf.internal." + "_message.OneofDescriptor"), // tp_name + sizeof(PyBaseDescriptor), // tp_basicsize + 0, // tp_itemsize + 0, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + C("A Oneof Descriptor"), // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + 0, // tp_methods + 0, // tp_members + oneof_descriptor::Getters, // tp_getset + &descriptor::PyBaseDescriptor_Type, // tp_base +}; + +PyObject* PyOneofDescriptor_New( + const OneofDescriptor* oneof_descriptor) { + return descriptor::NewInternedDescriptor( + &PyOneofDescriptor_Type, oneof_descriptor); +} + +// Add a enum values to a type dictionary. +static bool AddEnumValues(PyTypeObject *type, + const EnumDescriptor* enum_descriptor) { + for (int i = 0; i < enum_descriptor->value_count(); ++i) { + const EnumValueDescriptor* value = enum_descriptor->value(i); + ScopedPyObjectPtr obj(PyInt_FromLong(value->number())); + if (obj == NULL) { + return false; + } + if (PyDict_SetItemString(type->tp_dict, value->name().c_str(), obj) < 0) { + return false; + } } + return true; +} - Py_RETURN_NONE; +static bool AddIntConstant(PyTypeObject *type, const char* name, int value) { + ScopedPyObjectPtr obj(PyInt_FromLong(value)); + if (PyDict_SetItemString(type->tp_dict, name, obj) < 0) { + return false; + } + return true; } + bool InitDescriptor() { - if (PyType_Ready(&CMessageDescriptor_Type) < 0) + if (PyType_Ready(&PyMessageDescriptor_Type) < 0) + return false; + + if (PyType_Ready(&PyFieldDescriptor_Type) < 0) + return false; + + if (!AddEnumValues(&PyFieldDescriptor_Type, + FieldDescriptorProto::Label_descriptor())) { + return false; + } + if (!AddEnumValues(&PyFieldDescriptor_Type, + FieldDescriptorProto::Type_descriptor())) { + return false; + } +#define ADD_FIELDDESC_CONSTANT(NAME) AddIntConstant( \ + &PyFieldDescriptor_Type, #NAME, FieldDescriptor::NAME) + if (!ADD_FIELDDESC_CONSTANT(CPPTYPE_INT32) || + !ADD_FIELDDESC_CONSTANT(CPPTYPE_INT64) || + !ADD_FIELDDESC_CONSTANT(CPPTYPE_UINT32) || + !ADD_FIELDDESC_CONSTANT(CPPTYPE_UINT64) || + !ADD_FIELDDESC_CONSTANT(CPPTYPE_DOUBLE) || + !ADD_FIELDDESC_CONSTANT(CPPTYPE_FLOAT) || + !ADD_FIELDDESC_CONSTANT(CPPTYPE_BOOL) || + !ADD_FIELDDESC_CONSTANT(CPPTYPE_ENUM) || + !ADD_FIELDDESC_CONSTANT(CPPTYPE_STRING) || + !ADD_FIELDDESC_CONSTANT(CPPTYPE_MESSAGE)) { return false; - if (PyType_Ready(&CFieldDescriptor_Type) < 0) + } +#undef ADD_FIELDDESC_CONSTANT + + if (PyType_Ready(&PyEnumDescriptor_Type) < 0) + return false; + + if (PyType_Ready(&PyEnumValueDescriptor_Type) < 0) + return false; + + if (PyType_Ready(&PyFileDescriptor_Type) < 0) + return false; + + if (PyType_Ready(&PyOneofDescriptor_Type) < 0) return false; - PyDescriptorPool_Type.tp_new = PyType_GenericNew; - if (PyType_Ready(&PyDescriptorPool_Type) < 0) + if (!InitDescriptorMappingTypes()) return false; return true; diff --git a/python/google/protobuf/pyext/descriptor.h b/python/google/protobuf/pyext/descriptor.h index 9e5957b5..ba6e7298 100644 --- a/python/google/protobuf/pyext/descriptor.h +++ b/python/google/protobuf/pyext/descriptor.h @@ -34,105 +34,55 @@ #define GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_H__ #include <Python.h> -#include <structmember.h> - -#include <google/protobuf/stubs/hash.h> #include <google/protobuf/descriptor.h> -#if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN) -typedef int Py_ssize_t; -#define PY_SSIZE_T_MAX INT_MAX -#define PY_SSIZE_T_MIN INT_MIN -#endif - namespace google { namespace protobuf { namespace python { -typedef struct CMessageDescriptor { - PyObject_HEAD - - // The proto2 descriptor that this object represents. - const google::protobuf::Descriptor* descriptor; -} CMessageDescriptor; - - -typedef struct CFieldDescriptor { - PyObject_HEAD - - // The proto2 descriptor that this object represents. - const google::protobuf::FieldDescriptor* descriptor; -} CFieldDescriptor; - - -// Wraps operations to the global DescriptorPool which contains information -// about all messages and fields. -// -// There is normally one pool per process. We make it a Python object only -// because it contains many Python references. -// TODO(amauryfa): See whether such objects can appear in reference cycles, and -// consider adding support for the cyclic GC. -// -// "Methods" that interacts with this DescriptorPool are in the cdescriptor_pool -// namespace. -typedef struct PyDescriptorPool { - PyObject_HEAD - - google::protobuf::DescriptorPool* pool; - - // Make our own mapping to retrieve Python classes from C++ descriptors. - // - // Descriptor pointers stored here are owned by the DescriptorPool above. - // Python references to classes are owned by this PyDescriptorPool. - typedef hash_map<const Descriptor *, PyObject *> ClassesByMessageMap; - ClassesByMessageMap *classes_by_descriptor; -} PyDescriptorPool; - - -extern PyTypeObject CMessageDescriptor_Type; -extern PyTypeObject CFieldDescriptor_Type; - -extern PyTypeObject PyDescriptorPool_Type; - - -namespace cdescriptor_pool { - -// Builds a new DescriptorPool. Normally called only once per process. -PyDescriptorPool* NewDescriptorPool(); - -// Looks up a message by name. -// Returns a message Descriptor, or NULL if not found. -const google::protobuf::Descriptor* FindMessageTypeByName(PyDescriptorPool* self, - const string& name); - -// Registers a new Python class for the given message descriptor. -// Returns the message Descriptor. -// On error, returns NULL with a Python exception set. -const google::protobuf::Descriptor* RegisterMessageClass( - PyDescriptorPool* self, PyObject *message_class, PyObject *descriptor); +extern PyTypeObject PyMessageDescriptor_Type; +extern PyTypeObject PyFieldDescriptor_Type; +extern PyTypeObject PyEnumDescriptor_Type; +extern PyTypeObject PyEnumValueDescriptor_Type; +extern PyTypeObject PyFileDescriptor_Type; +extern PyTypeObject PyOneofDescriptor_Type; + +// Return a new reference to a Descriptor object. +// The C++ pointer is usually borrowed from the global DescriptorPool. +// In any case, it must stay alive as long as the Python object. +PyObject* PyMessageDescriptor_New(const Descriptor* descriptor); +PyObject* PyFieldDescriptor_New(const FieldDescriptor* descriptor); +PyObject* PyEnumDescriptor_New(const EnumDescriptor* descriptor); +PyObject* PyEnumValueDescriptor_New(const EnumValueDescriptor* descriptor); +PyObject* PyOneofDescriptor_New(const OneofDescriptor* descriptor); +PyObject* PyFileDescriptor_New(const FileDescriptor* file_descriptor); + +// Alternate constructor of PyFileDescriptor, used when we already have a +// serialized FileDescriptorProto that can be cached. +// Returns a new reference. +PyObject* PyFileDescriptor_NewWithPb(const FileDescriptor* file_descriptor, + PyObject* serialized_pb); -// Retrieves the Python class registered with the given message descriptor. -// -// Returns a *borrowed* reference if found, otherwise returns NULL with an +// Return the C++ descriptor pointer. +// This function checks the parameter type; on error, return NULL with a Python // exception set. -PyObject *GetMessageClass(PyDescriptorPool* self, - const Descriptor *message_descriptor); +const Descriptor* PyMessageDescriptor_AsDescriptor(PyObject* obj); +const FieldDescriptor* PyFieldDescriptor_AsDescriptor(PyObject* obj); -// Looks up a field by name. Returns a CDescriptor corresponding to -// the field on success, or NULL on failure. -// -// Returns a new reference. -PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* name); +// Returns the raw C++ pointer. +const void* PyDescriptor_AsVoidPtr(PyObject* obj); -// Looks up an extension by name. Returns a CDescriptor corresponding -// to the field on success, or NULL on failure. +// Check that the calling Python code is the global scope of a _pb2.py module. +// This function is used to support the current code generated by the proto +// compiler, which insists on modifying descriptors after they have been +// created. // -// Returns a new reference. -PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg); -} // namespace cdescriptor_pool +// stacklevel indicates which Python frame should be the _pb2.py module. +// +// Don't use this function outside descriptor classes. +bool _CalledFromGeneratedFile(int stacklevel); -PyObject* Python_BuildFile(PyObject* ignored, PyObject* args); bool InitDescriptor(); } // namespace python diff --git a/python/google/protobuf/pyext/descriptor_containers.cc b/python/google/protobuf/pyext/descriptor_containers.cc new file mode 100644 index 00000000..06edebf8 --- /dev/null +++ b/python/google/protobuf/pyext/descriptor_containers.cc @@ -0,0 +1,1564 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Mappings and Sequences of descriptors. +// Used by Descriptor.fields_by_name, EnumDescriptor.values... +// +// They avoid the allocation of a full dictionary or a full list: they simply +// store a pointer to the parent descriptor, use the C++ Descriptor methods (see +// google/protobuf/descriptor.h) to retrieve other descriptors, and create +// Python objects on the fly. +// +// The containers fully conform to abc.Mapping and abc.Sequence, and behave just +// like read-only dictionaries and lists. +// +// Because the interface of C++ Descriptors is quite regular, this file actually +// defines only three types, the exact behavior of a container is controlled by +// a DescriptorContainerDef structure, which contains functions that uses the +// public Descriptor API. +// +// Note: This DescriptorContainerDef is similar to the "virtual methods table" +// that a C++ compiler generates for a class. We have to make it explicit +// because the Python API is based on C, and does not play well with C++ +// inheritance. + +#include <Python.h> + +#include <google/protobuf/descriptor.h> +#include <google/protobuf/pyext/descriptor_containers.h> +#include <google/protobuf/pyext/descriptor_pool.h> +#include <google/protobuf/pyext/descriptor.h> +#include <google/protobuf/pyext/scoped_pyobject_ptr.h> + +#if PY_MAJOR_VERSION >= 3 + #define PyString_FromStringAndSize PyUnicode_FromStringAndSize + #define PyString_FromFormat PyUnicode_FromFormat + #define PyInt_FromLong PyLong_FromLong + #if PY_VERSION_HEX < 0x03030000 + #error "Python 3.0 - 3.2 are not supported." + #endif + #define PyString_AsStringAndSize(ob, charpp, sizep) \ + (PyUnicode_Check(ob)? \ + ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \ + PyBytes_AsStringAndSize(ob, (charpp), (sizep))) +#endif + +namespace google { +namespace protobuf { +namespace python { + +struct PyContainer; + +typedef int (*CountMethod)(PyContainer* self); +typedef const void* (*GetByIndexMethod)(PyContainer* self, int index); +typedef const void* (*GetByNameMethod)(PyContainer* self, const string& name); +typedef const void* (*GetByNumberMethod)(PyContainer* self, int index); +typedef PyObject* (*NewObjectFromItemMethod)(const void* descriptor); +typedef const string& (*GetItemNameMethod)(const void* descriptor); +typedef int (*GetItemNumberMethod)(const void* descriptor); +typedef int (*GetItemIndexMethod)(const void* descriptor); + +struct DescriptorContainerDef { + const char* mapping_name; + // Returns the number of items in the container. + CountMethod count_fn; + // Retrieve item by index (usually the order of declaration in the proto file) + // Used by sequences, but also iterators. 0 <= index < Count(). + GetByIndexMethod get_by_index_fn; + // Retrieve item by name (usually a call to some 'FindByName' method). + // Used by "by_name" mappings. + GetByNameMethod get_by_name_fn; + // Retrieve item by declared number (field tag, or enum value). + // Used by "by_number" mappings. + GetByNumberMethod get_by_number_fn; + // Converts a item C++ descriptor to a Python object. Returns a new reference. + NewObjectFromItemMethod new_object_from_item_fn; + // Retrieve the name of an item. Used by iterators on "by_name" mappings. + GetItemNameMethod get_item_name_fn; + // Retrieve the number of an item. Used by iterators on "by_number" mappings. + GetItemNumberMethod get_item_number_fn; + // Retrieve the index of an item for the container type. + // Used by "__contains__". + // If not set, "x in sequence" will do a linear search. + GetItemIndexMethod get_item_index_fn; +}; + +struct PyContainer { + PyObject_HEAD + + // The proto2 descriptor this container belongs to the global DescriptorPool. + const void* descriptor; + + // A pointer to a static structure with function pointers that control the + // behavior of the container. Very similar to the table of virtual functions + // of a C++ class. + const DescriptorContainerDef* container_def; + + // The kind of container: list, or dict by name or value. + enum ContainerKind { + KIND_SEQUENCE, + KIND_BYNAME, + KIND_BYNUMBER, + } kind; +}; + +struct PyContainerIterator { + PyObject_HEAD + + // The container we are iterating over. Own a reference. + PyContainer* container; + + // The current index in the iterator. + int index; + + // The kind of container: list, or dict by name or value. + enum IterKind { + KIND_ITERKEY, + KIND_ITERVALUE, + KIND_ITERITEM, + KIND_ITERVALUE_REVERSED, // For sequences + } kind; +}; + +namespace descriptor { + +// Returns the C++ item descriptor for a given Python key. +// When the descriptor is found, return true and set *item. +// When the descriptor is not found, return true, but set *item to NULL. +// On error, returns false with an exception set. +static bool _GetItemByKey(PyContainer* self, PyObject* key, const void** item) { + switch (self->kind) { + case PyContainer::KIND_BYNAME: + { + char* name; + Py_ssize_t name_size; + if (PyString_AsStringAndSize(key, &name, &name_size) < 0) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + // Not a string, cannot be in the container. + PyErr_Clear(); + *item = NULL; + return true; + } + return false; + } + *item = self->container_def->get_by_name_fn( + self, string(name, name_size)); + return true; + } + case PyContainer::KIND_BYNUMBER: + { + Py_ssize_t number = PyNumber_AsSsize_t(key, NULL); + if (number == -1 && PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + // Not a number, cannot be in the container. + PyErr_Clear(); + *item = NULL; + return true; + } + return false; + } + *item = self->container_def->get_by_number_fn(self, number); + return true; + } + default: + PyErr_SetNone(PyExc_NotImplementedError); + return false; + } +} + +// Returns the key of the object at the given index. +// Used when iterating over mappings. +static PyObject* _NewKey_ByIndex(PyContainer* self, Py_ssize_t index) { + const void* item = self->container_def->get_by_index_fn(self, index); + switch (self->kind) { + case PyContainer::KIND_BYNAME: + { + const string& name(self->container_def->get_item_name_fn(item)); + return PyString_FromStringAndSize(name.c_str(), name.size()); + } + case PyContainer::KIND_BYNUMBER: + { + int value = self->container_def->get_item_number_fn(item); + return PyInt_FromLong(value); + } + default: + PyErr_SetNone(PyExc_NotImplementedError); + return NULL; + } +} + +// Returns the object at the given index. +// Also used when iterating over mappings. +static PyObject* _NewObj_ByIndex(PyContainer* self, Py_ssize_t index) { + return self->container_def->new_object_from_item_fn( + self->container_def->get_by_index_fn(self, index)); +} + +static Py_ssize_t Length(PyContainer* self) { + return self->container_def->count_fn(self); +} + +// The DescriptorMapping type. + +static PyObject* Subscript(PyContainer* self, PyObject* key) { + const void* item = NULL; + if (!_GetItemByKey(self, key, &item)) { + return NULL; + } + if (!item) { + PyErr_SetObject(PyExc_KeyError, key); + return NULL; + } + return self->container_def->new_object_from_item_fn(item); +} + +static int AssSubscript(PyContainer* self, PyObject* key, PyObject* value) { + if (_CalledFromGeneratedFile(0)) { + return 0; + } + PyErr_Format(PyExc_TypeError, + "'%.200s' object does not support item assignment", + Py_TYPE(self)->tp_name); + return -1; +} + +static PyMappingMethods MappingMappingMethods = { + (lenfunc)Length, // mp_length + (binaryfunc)Subscript, // mp_subscript + (objobjargproc)AssSubscript, // mp_ass_subscript +}; + +static int Contains(PyContainer* self, PyObject* key) { + const void* item = NULL; + if (!_GetItemByKey(self, key, &item)) { + return -1; + } + if (item) { + return 1; + } else { + return 0; + } +} + +static PyObject* ContainerRepr(PyContainer* self) { + const char* kind = ""; + switch (self->kind) { + case PyContainer::KIND_SEQUENCE: + kind = "sequence"; + break; + case PyContainer::KIND_BYNAME: + kind = "mapping by name"; + break; + case PyContainer::KIND_BYNUMBER: + kind = "mapping by number"; + break; + } + return PyString_FromFormat( + "<%s %s>", self->container_def->mapping_name, kind); +} + +extern PyTypeObject DescriptorMapping_Type; +extern PyTypeObject DescriptorSequence_Type; + +// A sequence container can only be equal to another sequence container, or (for +// backward compatibility) to a list containing the same items. +// Returns 1 if equal, 0 if unequal, -1 on error. +static int DescriptorSequence_Equal(PyContainer* self, PyObject* other) { + // Check the identity of C++ pointers. + if (PyObject_TypeCheck(other, &DescriptorSequence_Type)) { + PyContainer* other_container = reinterpret_cast<PyContainer*>(other); + if (self->descriptor == other_container->descriptor && + self->container_def == other_container->container_def && + self->kind == other_container->kind) { + return 1; + } else { + return 0; + } + } + + // If other is a list + if (PyList_Check(other)) { + // return list(self) == other + int size = Length(self); + if (size != PyList_Size(other)) { + return false; + } + for (int index = 0; index < size; index++) { + ScopedPyObjectPtr value1(_NewObj_ByIndex(self, index)); + if (value1 == NULL) { + return -1; + } + PyObject* value2 = PyList_GetItem(other, index); + if (value2 == NULL) { + return -1; + } + int cmp = PyObject_RichCompareBool(value1, value2, Py_EQ); + if (cmp != 1) // error or not equal + return cmp; + } + // All items were found and equal + return 1; + } + + // Any other object is different. + return 0; +} + +// A mapping container can only be equal to another mapping container, or (for +// backward compatibility) to a dict containing the same items. +// Returns 1 if equal, 0 if unequal, -1 on error. +static int DescriptorMapping_Equal(PyContainer* self, PyObject* other) { + // Check the identity of C++ pointers. + if (PyObject_TypeCheck(other, &DescriptorMapping_Type)) { + PyContainer* other_container = reinterpret_cast<PyContainer*>(other); + if (self->descriptor == other_container->descriptor && + self->container_def == other_container->container_def && + self->kind == other_container->kind) { + return 1; + } else { + return 0; + } + } + + // If other is a dict + if (PyDict_Check(other)) { + // equivalent to dict(self.items()) == other + int size = Length(self); + if (size != PyDict_Size(other)) { + return false; + } + for (int index = 0; index < size; index++) { + ScopedPyObjectPtr key(_NewKey_ByIndex(self, index)); + if (key == NULL) { + return -1; + } + ScopedPyObjectPtr value1(_NewObj_ByIndex(self, index)); + if (value1 == NULL) { + return -1; + } + PyObject* value2 = PyDict_GetItem(other, key); + if (value2 == NULL) { + // Not found in the other dictionary + return 0; + } + int cmp = PyObject_RichCompareBool(value1, value2, Py_EQ); + if (cmp != 1) // error or not equal + return cmp; + } + // All items were found and equal + return 1; + } + + // Any other object is different. + return 0; +} + +static PyObject* RichCompare(PyContainer* self, PyObject* other, int opid) { + if (opid != Py_EQ && opid != Py_NE) { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + + int result; + + if (self->kind == PyContainer::KIND_SEQUENCE) { + result = DescriptorSequence_Equal(self, other); + } else { + result = DescriptorMapping_Equal(self, other); + } + if (result < 0) { + return NULL; + } + if (result ^ (opid == Py_NE)) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } +} + +static PySequenceMethods MappingSequenceMethods = { + 0, // sq_length + 0, // sq_concat + 0, // sq_repeat + 0, // sq_item + 0, // sq_slice + 0, // sq_ass_item + 0, // sq_ass_slice + (objobjproc)Contains, // sq_contains +}; + +static PyObject* Get(PyContainer* self, PyObject* args) { + PyObject* key; + PyObject* default_value = Py_None; + if (!PyArg_UnpackTuple(args, "get", 1, 2, &key, &default_value)) { + return NULL; + } + + const void* item; + if (!_GetItemByKey(self, key, &item)) { + return NULL; + } + if (item == NULL) { + Py_INCREF(default_value); + return default_value; + } + return self->container_def->new_object_from_item_fn(item); +} + +static PyObject* Keys(PyContainer* self, PyObject* args) { + Py_ssize_t count = Length(self); + ScopedPyObjectPtr list(PyList_New(count)); + if (list == NULL) { + return NULL; + } + for (Py_ssize_t index = 0; index < count; ++index) { + PyObject* key = _NewKey_ByIndex(self, index); + if (key == NULL) { + return NULL; + } + PyList_SET_ITEM(list.get(), index, key); + } + return list.release(); +} + +static PyObject* Values(PyContainer* self, PyObject* args) { + Py_ssize_t count = Length(self); + ScopedPyObjectPtr list(PyList_New(count)); + if (list == NULL) { + return NULL; + } + for (Py_ssize_t index = 0; index < count; ++index) { + PyObject* value = _NewObj_ByIndex(self, index); + if (value == NULL) { + return NULL; + } + PyList_SET_ITEM(list.get(), index, value); + } + return list.release(); +} + +static PyObject* Items(PyContainer* self, PyObject* args) { + Py_ssize_t count = Length(self); + ScopedPyObjectPtr list(PyList_New(count)); + if (list == NULL) { + return NULL; + } + for (Py_ssize_t index = 0; index < count; ++index) { + ScopedPyObjectPtr obj(PyTuple_New(2)); + if (obj == NULL) { + return NULL; + } + PyObject* key = _NewKey_ByIndex(self, index); + if (key == NULL) { + return NULL; + } + PyTuple_SET_ITEM(obj.get(), 0, key); + PyObject* value = _NewObj_ByIndex(self, index); + if (value == NULL) { + return NULL; + } + PyTuple_SET_ITEM(obj.get(), 1, value); + PyList_SET_ITEM(list.get(), index, obj.release()); + } + return list.release(); +} + +static PyObject* NewContainerIterator(PyContainer* mapping, + PyContainerIterator::IterKind kind); + +static PyObject* Iter(PyContainer* self) { + return NewContainerIterator(self, PyContainerIterator::KIND_ITERKEY); +} +static PyObject* IterKeys(PyContainer* self, PyObject* args) { + return NewContainerIterator(self, PyContainerIterator::KIND_ITERKEY); +} +static PyObject* IterValues(PyContainer* self, PyObject* args) { + return NewContainerIterator(self, PyContainerIterator::KIND_ITERVALUE); +} +static PyObject* IterItems(PyContainer* self, PyObject* args) { + return NewContainerIterator(self, PyContainerIterator::KIND_ITERITEM); +} + +static PyMethodDef MappingMethods[] = { + { "get", (PyCFunction)Get, METH_VARARGS, }, + { "keys", (PyCFunction)Keys, METH_NOARGS, }, + { "values", (PyCFunction)Values, METH_NOARGS, }, + { "items", (PyCFunction)Items, METH_NOARGS, }, + { "iterkeys", (PyCFunction)IterKeys, METH_NOARGS, }, + { "itervalues", (PyCFunction)IterValues, METH_NOARGS, }, + { "iteritems", (PyCFunction)IterItems, METH_NOARGS, }, + {NULL} +}; + +PyTypeObject DescriptorMapping_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "DescriptorMapping", // tp_name + sizeof(PyContainer), // tp_basicsize + 0, // tp_itemsize + 0, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + (reprfunc)ContainerRepr, // tp_repr + 0, // tp_as_number + &MappingSequenceMethods, // tp_as_sequence + &MappingMappingMethods, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + 0, // tp_doc + 0, // tp_traverse + 0, // tp_clear + (richcmpfunc)RichCompare, // tp_richcompare + 0, // tp_weaklistoffset + (getiterfunc)Iter, // tp_iter + 0, // tp_iternext + MappingMethods, // tp_methods + 0, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init + 0, // tp_alloc + 0, // tp_new + 0, // tp_free +}; + +// The DescriptorSequence type. + +static PyObject* GetItem(PyContainer* self, Py_ssize_t index) { + if (index < 0) { + index += Length(self); + } + if (index < 0 || index >= Length(self)) { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return NULL; + } + return _NewObj_ByIndex(self, index); +} + +// Returns the position of the item in the sequence, of -1 if not found. +// This function never fails. +int Find(PyContainer* self, PyObject* item) { + // The item can only be in one position: item.index. + // Check that self[item.index] == item, it's faster than a linear search. + // + // This assumes that sequences are only defined by syntax of the .proto file: + // a specific item belongs to only one sequence, depending on its position in + // the .proto file definition. + const void* descriptor_ptr = PyDescriptor_AsVoidPtr(item); + if (descriptor_ptr == NULL) { + // Not a descriptor, it cannot be in the list. + return -1; + } + if (self->container_def->get_item_index_fn) { + int index = self->container_def->get_item_index_fn(descriptor_ptr); + if (index < 0 || index >= Length(self)) { + // This index is not from this collection. + return -1; + } + if (self->container_def->get_by_index_fn(self, index) != descriptor_ptr) { + // The descriptor at this index is not the same. + return -1; + } + // self[item.index] == item, so return the index. + return index; + } else { + // Fall back to linear search. + int length = Length(self); + for (int index=0; index < length; index++) { + if (self->container_def->get_by_index_fn(self, index) == descriptor_ptr) { + return index; + } + } + // Not found + return -1; + } +} + +// Implements list.index(): the position of the item is in the sequence. +static PyObject* Index(PyContainer* self, PyObject* item) { + int position = Find(self, item); + if (position < 0) { + // Not found + PyErr_SetNone(PyExc_ValueError); + return NULL; + } else { + return PyInt_FromLong(position); + } +} +// Implements "list.__contains__()": is the object in the sequence. +static int SeqContains(PyContainer* self, PyObject* item) { + int position = Find(self, item); + if (position < 0) { + return 0; + } else { + return 1; + } +} + +// Implements list.count(): number of occurrences of the item in the sequence. +// An item can only appear once in a sequence. If it exists, return 1. +static PyObject* Count(PyContainer* self, PyObject* item) { + int position = Find(self, item); + if (position < 0) { + return PyInt_FromLong(0); + } else { + return PyInt_FromLong(1); + } +} + +static PyObject* Append(PyContainer* self, PyObject* args) { + if (_CalledFromGeneratedFile(0)) { + Py_RETURN_NONE; + } + PyErr_Format(PyExc_TypeError, + "'%.200s' object is not a mutable sequence", + Py_TYPE(self)->tp_name); + return NULL; +} + +static PyObject* Reversed(PyContainer* self, PyObject* args) { + return NewContainerIterator(self, + PyContainerIterator::KIND_ITERVALUE_REVERSED); +} + +static PyMethodDef SeqMethods[] = { + { "index", (PyCFunction)Index, METH_O, }, + { "count", (PyCFunction)Count, METH_O, }, + { "append", (PyCFunction)Append, METH_O, }, + { "__reversed__", (PyCFunction)Reversed, METH_NOARGS, }, + {NULL} +}; + +static PySequenceMethods SeqSequenceMethods = { + (lenfunc)Length, // sq_length + 0, // sq_concat + 0, // sq_repeat + (ssizeargfunc)GetItem, // sq_item + 0, // sq_slice + 0, // sq_ass_item + 0, // sq_ass_slice + (objobjproc)SeqContains, // sq_contains +}; + +PyTypeObject DescriptorSequence_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "DescriptorSequence", // tp_name + sizeof(PyContainer), // tp_basicsize + 0, // tp_itemsize + 0, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + (reprfunc)ContainerRepr, // tp_repr + 0, // tp_as_number + &SeqSequenceMethods, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + 0, // tp_doc + 0, // tp_traverse + 0, // tp_clear + (richcmpfunc)RichCompare, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + SeqMethods, // tp_methods + 0, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init + 0, // tp_alloc + 0, // tp_new + 0, // tp_free +}; + +static PyObject* NewMappingByName( + DescriptorContainerDef* container_def, const void* descriptor) { + PyContainer* self = PyObject_New(PyContainer, &DescriptorMapping_Type); + if (self == NULL) { + return NULL; + } + self->descriptor = descriptor; + self->container_def = container_def; + self->kind = PyContainer::KIND_BYNAME; + return reinterpret_cast<PyObject*>(self); +} + +static PyObject* NewMappingByNumber( + DescriptorContainerDef* container_def, const void* descriptor) { + if (container_def->get_by_number_fn == NULL || + container_def->get_item_number_fn == NULL) { + PyErr_SetNone(PyExc_NotImplementedError); + return NULL; + } + PyContainer* self = PyObject_New(PyContainer, &DescriptorMapping_Type); + if (self == NULL) { + return NULL; + } + self->descriptor = descriptor; + self->container_def = container_def; + self->kind = PyContainer::KIND_BYNUMBER; + return reinterpret_cast<PyObject*>(self); +} + +static PyObject* NewSequence( + DescriptorContainerDef* container_def, const void* descriptor) { + PyContainer* self = PyObject_New(PyContainer, &DescriptorSequence_Type); + if (self == NULL) { + return NULL; + } + self->descriptor = descriptor; + self->container_def = container_def; + self->kind = PyContainer::KIND_SEQUENCE; + return reinterpret_cast<PyObject*>(self); +} + +// Implement iterators over PyContainers. + +static void Iterator_Dealloc(PyContainerIterator* self) { + Py_CLEAR(self->container); + Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self)); +} + +static PyObject* Iterator_Next(PyContainerIterator* self) { + int count = self->container->container_def->count_fn(self->container); + if (self->index >= count) { + // Return NULL with no exception to indicate the end. + return NULL; + } + int index = self->index; + self->index += 1; + switch (self->kind) { + case PyContainerIterator::KIND_ITERKEY: + return _NewKey_ByIndex(self->container, index); + case PyContainerIterator::KIND_ITERVALUE: + return _NewObj_ByIndex(self->container, index); + case PyContainerIterator::KIND_ITERVALUE_REVERSED: + return _NewObj_ByIndex(self->container, count - index - 1); + case PyContainerIterator::KIND_ITERITEM: + { + PyObject* obj = PyTuple_New(2); + if (obj == NULL) { + return NULL; + } + PyObject* key = _NewKey_ByIndex(self->container, index); + if (key == NULL) { + Py_DECREF(obj); + return NULL; + } + PyTuple_SET_ITEM(obj, 0, key); + PyObject* value = _NewObj_ByIndex(self->container, index); + if (value == NULL) { + Py_DECREF(obj); + return NULL; + } + PyTuple_SET_ITEM(obj, 1, value); + return obj; + } + default: + PyErr_SetNone(PyExc_NotImplementedError); + return NULL; + } +} + +static PyTypeObject ContainerIterator_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "DescriptorContainerIterator", // tp_name + sizeof(PyContainerIterator), // tp_basicsize + 0, // tp_itemsize + (destructor)Iterator_Dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + 0, // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + PyObject_SelfIter, // tp_iter + (iternextfunc)Iterator_Next, // tp_iternext + 0, // tp_methods + 0, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init + 0, // tp_alloc + 0, // tp_new + 0, // tp_free +}; + +static PyObject* NewContainerIterator(PyContainer* container, + PyContainerIterator::IterKind kind) { + PyContainerIterator* self = PyObject_New(PyContainerIterator, + &ContainerIterator_Type); + if (self == NULL) { + return NULL; + } + Py_INCREF(container); + self->container = container; + self->kind = kind; + self->index = 0; + + return reinterpret_cast<PyObject*>(self); +} + +} // namespace descriptor + +// Now define the real collections! + +namespace message_descriptor { + +typedef const Descriptor* ParentDescriptor; + +static ParentDescriptor GetDescriptor(PyContainer* self) { + return reinterpret_cast<ParentDescriptor>(self->descriptor); +} + +namespace fields { + +typedef const FieldDescriptor* ItemDescriptor; + +static int Count(PyContainer* self) { + return GetDescriptor(self)->field_count(); +} + +static ItemDescriptor GetByName(PyContainer* self, const string& name) { + return GetDescriptor(self)->FindFieldByName(name); +} + +static ItemDescriptor GetByNumber(PyContainer* self, int number) { + return GetDescriptor(self)->FindFieldByNumber(number); +} + +static ItemDescriptor GetByIndex(PyContainer* self, int index) { + return GetDescriptor(self)->field(index); +} + +static PyObject* NewObjectFromItem(ItemDescriptor item) { + return PyFieldDescriptor_New(item); +} + +static const string& GetItemName(ItemDescriptor item) { + return item->name(); +} + +static int GetItemNumber(ItemDescriptor item) { + return item->number(); +} + +static int GetItemIndex(ItemDescriptor item) { + return item->index(); +} + +static DescriptorContainerDef ContainerDef = { + "MessageFields", + (CountMethod)Count, + (GetByIndexMethod)GetByIndex, + (GetByNameMethod)GetByName, + (GetByNumberMethod)GetByNumber, + (NewObjectFromItemMethod)NewObjectFromItem, + (GetItemNameMethod)GetItemName, + (GetItemNumberMethod)GetItemNumber, + (GetItemIndexMethod)GetItemIndex, +}; + +} // namespace fields + +PyObject* NewMessageFieldsByName(ParentDescriptor descriptor) { + return descriptor::NewMappingByName(&fields::ContainerDef, descriptor); +} + +PyObject* NewMessageFieldsByNumber(ParentDescriptor descriptor) { + return descriptor::NewMappingByNumber(&fields::ContainerDef, descriptor); +} + +PyObject* NewMessageFieldsSeq(ParentDescriptor descriptor) { + return descriptor::NewSequence(&fields::ContainerDef, descriptor); +} + +namespace nested_types { + +typedef const Descriptor* ItemDescriptor; + +static int Count(PyContainer* self) { + return GetDescriptor(self)->nested_type_count(); +} + +static ItemDescriptor GetByName(PyContainer* self, const string& name) { + return GetDescriptor(self)->FindNestedTypeByName(name); +} + +static ItemDescriptor GetByIndex(PyContainer* self, int index) { + return GetDescriptor(self)->nested_type(index); +} + +static PyObject* NewObjectFromItem(ItemDescriptor item) { + return PyMessageDescriptor_New(item); +} + +static const string& GetItemName(ItemDescriptor item) { + return item->name(); +} + +static int GetItemIndex(ItemDescriptor item) { + return item->index(); +} + +static DescriptorContainerDef ContainerDef = { + "MessageNestedTypes", + (CountMethod)Count, + (GetByIndexMethod)GetByIndex, + (GetByNameMethod)GetByName, + (GetByNumberMethod)NULL, + (NewObjectFromItemMethod)NewObjectFromItem, + (GetItemNameMethod)GetItemName, + (GetItemNumberMethod)NULL, + (GetItemIndexMethod)GetItemIndex, +}; + +} // namespace nested_types + +PyObject* NewMessageNestedTypesSeq(ParentDescriptor descriptor) { + return descriptor::NewSequence(&nested_types::ContainerDef, descriptor); +} + +PyObject* NewMessageNestedTypesByName(ParentDescriptor descriptor) { + return descriptor::NewMappingByName(&nested_types::ContainerDef, descriptor); +} + +namespace enums { + +typedef const EnumDescriptor* ItemDescriptor; + +static int Count(PyContainer* self) { + return GetDescriptor(self)->enum_type_count(); +} + +static ItemDescriptor GetByName(PyContainer* self, const string& name) { + return GetDescriptor(self)->FindEnumTypeByName(name); +} + +static ItemDescriptor GetByIndex(PyContainer* self, int index) { + return GetDescriptor(self)->enum_type(index); +} + +static PyObject* NewObjectFromItem(ItemDescriptor item) { + return PyEnumDescriptor_New(item); +} + +static const string& GetItemName(ItemDescriptor item) { + return item->name(); +} + +static int GetItemIndex(ItemDescriptor item) { + return item->index(); +} + +static DescriptorContainerDef ContainerDef = { + "MessageNestedEnums", + (CountMethod)Count, + (GetByIndexMethod)GetByIndex, + (GetByNameMethod)GetByName, + (GetByNumberMethod)NULL, + (NewObjectFromItemMethod)NewObjectFromItem, + (GetItemNameMethod)GetItemName, + (GetItemNumberMethod)NULL, + (GetItemIndexMethod)GetItemIndex, +}; + +} // namespace enums + +PyObject* NewMessageEnumsByName(ParentDescriptor descriptor) { + return descriptor::NewMappingByName(&enums::ContainerDef, descriptor); +} + +PyObject* NewMessageEnumsSeq(ParentDescriptor descriptor) { + return descriptor::NewSequence(&enums::ContainerDef, descriptor); +} + +namespace enumvalues { + +// This is the "enum_values_by_name" mapping, which collects values from all +// enum types in a message. +// +// Note that the behavior of the C++ descriptor is different: it will search and +// return the first value that matches the name, whereas the Python +// implementation retrieves the last one. + +typedef const EnumValueDescriptor* ItemDescriptor; + +static int Count(PyContainer* self) { + int count = 0; + for (int i = 0; i < GetDescriptor(self)->enum_type_count(); ++i) { + count += GetDescriptor(self)->enum_type(i)->value_count(); + } + return count; +} + +static ItemDescriptor GetByName(PyContainer* self, const string& name) { + return GetDescriptor(self)->FindEnumValueByName(name); +} + +static ItemDescriptor GetByIndex(PyContainer* self, int index) { + // This is not optimal, but the number of enums *types* in a given message + // is small. This function is only used when iterating over the mapping. + const EnumDescriptor* enum_type = NULL; + int enum_type_count = GetDescriptor(self)->enum_type_count(); + for (int i = 0; i < enum_type_count; ++i) { + enum_type = GetDescriptor(self)->enum_type(i); + int enum_value_count = enum_type->value_count(); + if (index < enum_value_count) { + // Found it! + break; + } + index -= enum_value_count; + } + // The next statement cannot overflow, because this function is only called by + // internal iterators which ensure that 0 <= index < Count(). + return enum_type->value(index); +} + +static PyObject* NewObjectFromItem(ItemDescriptor item) { + return PyEnumValueDescriptor_New(item); +} + +static const string& GetItemName(ItemDescriptor item) { + return item->name(); +} + +static DescriptorContainerDef ContainerDef = { + "MessageEnumValues", + (CountMethod)Count, + (GetByIndexMethod)GetByIndex, + (GetByNameMethod)GetByName, + (GetByNumberMethod)NULL, + (NewObjectFromItemMethod)NewObjectFromItem, + (GetItemNameMethod)GetItemName, + (GetItemNumberMethod)NULL, + (GetItemIndexMethod)NULL, +}; + +} // namespace enumvalues + +PyObject* NewMessageEnumValuesByName(ParentDescriptor descriptor) { + return descriptor::NewMappingByName(&enumvalues::ContainerDef, descriptor); +} + +namespace extensions { + +typedef const FieldDescriptor* ItemDescriptor; + +static int Count(PyContainer* self) { + return GetDescriptor(self)->extension_count(); +} + +static ItemDescriptor GetByName(PyContainer* self, const string& name) { + return GetDescriptor(self)->FindExtensionByName(name); +} + +static ItemDescriptor GetByIndex(PyContainer* self, int index) { + return GetDescriptor(self)->extension(index); +} + +static PyObject* NewObjectFromItem(ItemDescriptor item) { + return PyFieldDescriptor_New(item); +} + +static const string& GetItemName(ItemDescriptor item) { + return item->name(); +} + +static int GetItemIndex(ItemDescriptor item) { + return item->index(); +} + +static DescriptorContainerDef ContainerDef = { + "MessageExtensions", + (CountMethod)Count, + (GetByIndexMethod)GetByIndex, + (GetByNameMethod)GetByName, + (GetByNumberMethod)NULL, + (NewObjectFromItemMethod)NewObjectFromItem, + (GetItemNameMethod)GetItemName, + (GetItemNumberMethod)NULL, + (GetItemIndexMethod)GetItemIndex, +}; + +} // namespace extensions + +PyObject* NewMessageExtensionsByName(ParentDescriptor descriptor) { + return descriptor::NewMappingByName(&extensions::ContainerDef, descriptor); +} + +PyObject* NewMessageExtensionsSeq(ParentDescriptor descriptor) { + return descriptor::NewSequence(&extensions::ContainerDef, descriptor); +} + +namespace oneofs { + +typedef const OneofDescriptor* ItemDescriptor; + +static int Count(PyContainer* self) { + return GetDescriptor(self)->oneof_decl_count(); +} + +static ItemDescriptor GetByName(PyContainer* self, const string& name) { + return GetDescriptor(self)->FindOneofByName(name); +} + +static ItemDescriptor GetByIndex(PyContainer* self, int index) { + return GetDescriptor(self)->oneof_decl(index); +} + +static PyObject* NewObjectFromItem(ItemDescriptor item) { + return PyOneofDescriptor_New(item); +} + +static const string& GetItemName(ItemDescriptor item) { + return item->name(); +} + +static int GetItemIndex(ItemDescriptor item) { + return item->index(); +} + +static DescriptorContainerDef ContainerDef = { + "MessageOneofs", + (CountMethod)Count, + (GetByIndexMethod)GetByIndex, + (GetByNameMethod)GetByName, + (GetByNumberMethod)NULL, + (NewObjectFromItemMethod)NewObjectFromItem, + (GetItemNameMethod)GetItemName, + (GetItemNumberMethod)NULL, + (GetItemIndexMethod)GetItemIndex, +}; + +} // namespace oneofs + +PyObject* NewMessageOneofsByName(ParentDescriptor descriptor) { + return descriptor::NewMappingByName(&oneofs::ContainerDef, descriptor); +} + +PyObject* NewMessageOneofsSeq(ParentDescriptor descriptor) { + return descriptor::NewSequence(&oneofs::ContainerDef, descriptor); +} + +} // namespace message_descriptor + +namespace enum_descriptor { + +typedef const EnumDescriptor* ParentDescriptor; + +static ParentDescriptor GetDescriptor(PyContainer* self) { + return reinterpret_cast<ParentDescriptor>(self->descriptor); +} + +namespace enumvalues { + +typedef const EnumValueDescriptor* ItemDescriptor; + +static int Count(PyContainer* self) { + return GetDescriptor(self)->value_count(); +} + +static ItemDescriptor GetByIndex(PyContainer* self, int index) { + return GetDescriptor(self)->value(index); +} + +static ItemDescriptor GetByName(PyContainer* self, const string& name) { + return GetDescriptor(self)->FindValueByName(name); +} + +static ItemDescriptor GetByNumber(PyContainer* self, int number) { + return GetDescriptor(self)->FindValueByNumber(number); +} + +static PyObject* NewObjectFromItem(ItemDescriptor item) { + return PyEnumValueDescriptor_New(item); +} + +static const string& GetItemName(ItemDescriptor item) { + return item->name(); +} + +static int GetItemNumber(ItemDescriptor item) { + return item->number(); +} + +static int GetItemIndex(ItemDescriptor item) { + return item->index(); +} + +static DescriptorContainerDef ContainerDef = { + "EnumValues", + (CountMethod)Count, + (GetByIndexMethod)GetByIndex, + (GetByNameMethod)GetByName, + (GetByNumberMethod)GetByNumber, + (NewObjectFromItemMethod)NewObjectFromItem, + (GetItemNameMethod)GetItemName, + (GetItemNumberMethod)GetItemNumber, + (GetItemIndexMethod)GetItemIndex, +}; + +} // namespace enumvalues + +PyObject* NewEnumValuesByName(ParentDescriptor descriptor) { + return descriptor::NewMappingByName(&enumvalues::ContainerDef, descriptor); +} + +PyObject* NewEnumValuesByNumber(ParentDescriptor descriptor) { + return descriptor::NewMappingByNumber(&enumvalues::ContainerDef, descriptor); +} + +PyObject* NewEnumValuesSeq(ParentDescriptor descriptor) { + return descriptor::NewSequence(&enumvalues::ContainerDef, descriptor); +} + +} // namespace enum_descriptor + +namespace oneof_descriptor { + +typedef const OneofDescriptor* ParentDescriptor; + +static ParentDescriptor GetDescriptor(PyContainer* self) { + return reinterpret_cast<ParentDescriptor>(self->descriptor); +} + +namespace fields { + +typedef const FieldDescriptor* ItemDescriptor; + +static int Count(PyContainer* self) { + return GetDescriptor(self)->field_count(); +} + +static ItemDescriptor GetByIndex(PyContainer* self, int index) { + return GetDescriptor(self)->field(index); +} + +static PyObject* NewObjectFromItem(ItemDescriptor item) { + return PyFieldDescriptor_New(item); +} + +static int GetItemIndex(ItemDescriptor item) { + return item->index_in_oneof(); +} + +static DescriptorContainerDef ContainerDef = { + "OneofFields", + (CountMethod)Count, + (GetByIndexMethod)GetByIndex, + (GetByNameMethod)NULL, + (GetByNumberMethod)NULL, + (NewObjectFromItemMethod)NewObjectFromItem, + (GetItemNameMethod)NULL, + (GetItemNumberMethod)NULL, + (GetItemIndexMethod)GetItemIndex, +}; + +} // namespace fields + +PyObject* NewOneofFieldsSeq(ParentDescriptor descriptor) { + return descriptor::NewSequence(&fields::ContainerDef, descriptor); +} + +} // namespace oneof_descriptor + +namespace file_descriptor { + +typedef const FileDescriptor* ParentDescriptor; + +static ParentDescriptor GetDescriptor(PyContainer* self) { + return reinterpret_cast<ParentDescriptor>(self->descriptor); +} + +namespace messages { + +typedef const Descriptor* ItemDescriptor; + +static int Count(PyContainer* self) { + return GetDescriptor(self)->message_type_count(); +} + +static ItemDescriptor GetByName(PyContainer* self, const string& name) { + return GetDescriptor(self)->FindMessageTypeByName(name); +} + +static ItemDescriptor GetByIndex(PyContainer* self, int index) { + return GetDescriptor(self)->message_type(index); +} + +static PyObject* NewObjectFromItem(ItemDescriptor item) { + return PyMessageDescriptor_New(item); +} + +static const string& GetItemName(ItemDescriptor item) { + return item->name(); +} + +static int GetItemIndex(ItemDescriptor item) { + return item->index(); +} + +static DescriptorContainerDef ContainerDef = { + "FileMessages", + (CountMethod)Count, + (GetByIndexMethod)GetByIndex, + (GetByNameMethod)GetByName, + (GetByNumberMethod)NULL, + (NewObjectFromItemMethod)NewObjectFromItem, + (GetItemNameMethod)GetItemName, + (GetItemNumberMethod)NULL, + (GetItemIndexMethod)GetItemIndex, +}; + +} // namespace messages + +PyObject* NewFileMessageTypesByName(const FileDescriptor* descriptor) { + return descriptor::NewMappingByName(&messages::ContainerDef, descriptor); +} + +namespace enums { + +typedef const EnumDescriptor* ItemDescriptor; + +static int Count(PyContainer* self) { + return GetDescriptor(self)->enum_type_count(); +} + +static ItemDescriptor GetByName(PyContainer* self, const string& name) { + return GetDescriptor(self)->FindEnumTypeByName(name); +} + +static ItemDescriptor GetByIndex(PyContainer* self, int index) { + return GetDescriptor(self)->enum_type(index); +} + +static PyObject* NewObjectFromItem(ItemDescriptor item) { + return PyEnumDescriptor_New(item); +} + +static const string& GetItemName(ItemDescriptor item) { + return item->name(); +} + +static int GetItemIndex(ItemDescriptor item) { + return item->index(); +} + +static DescriptorContainerDef ContainerDef = { + "FileEnums", + (CountMethod)Count, + (GetByIndexMethod)GetByIndex, + (GetByNameMethod)GetByName, + (GetByNumberMethod)NULL, + (NewObjectFromItemMethod)NewObjectFromItem, + (GetItemNameMethod)GetItemName, + (GetItemNumberMethod)NULL, + (GetItemIndexMethod)GetItemIndex, +}; + +} // namespace enums + +PyObject* NewFileEnumTypesByName(const FileDescriptor* descriptor) { + return descriptor::NewMappingByName(&enums::ContainerDef, descriptor); +} + +namespace extensions { + +typedef const FieldDescriptor* ItemDescriptor; + +static int Count(PyContainer* self) { + return GetDescriptor(self)->extension_count(); +} + +static ItemDescriptor GetByName(PyContainer* self, const string& name) { + return GetDescriptor(self)->FindExtensionByName(name); +} + +static ItemDescriptor GetByIndex(PyContainer* self, int index) { + return GetDescriptor(self)->extension(index); +} + +static PyObject* NewObjectFromItem(ItemDescriptor item) { + return PyFieldDescriptor_New(item); +} + +static const string& GetItemName(ItemDescriptor item) { + return item->name(); +} + +static int GetItemIndex(ItemDescriptor item) { + return item->index(); +} + +static DescriptorContainerDef ContainerDef = { + "FileExtensions", + (CountMethod)Count, + (GetByIndexMethod)GetByIndex, + (GetByNameMethod)GetByName, + (GetByNumberMethod)NULL, + (NewObjectFromItemMethod)NewObjectFromItem, + (GetItemNameMethod)GetItemName, + (GetItemNumberMethod)NULL, + (GetItemIndexMethod)GetItemIndex, +}; + +} // namespace extensions + +PyObject* NewFileExtensionsByName(const FileDescriptor* descriptor) { + return descriptor::NewMappingByName(&extensions::ContainerDef, descriptor); +} + +namespace dependencies { + +typedef const FileDescriptor* ItemDescriptor; + +static int Count(PyContainer* self) { + return GetDescriptor(self)->dependency_count(); +} + +static ItemDescriptor GetByIndex(PyContainer* self, int index) { + return GetDescriptor(self)->dependency(index); +} + +static PyObject* NewObjectFromItem(ItemDescriptor item) { + return PyFileDescriptor_New(item); +} + +static DescriptorContainerDef ContainerDef = { + "FileDependencies", + (CountMethod)Count, + (GetByIndexMethod)GetByIndex, + (GetByNameMethod)NULL, + (GetByNumberMethod)NULL, + (NewObjectFromItemMethod)NewObjectFromItem, + (GetItemNameMethod)NULL, + (GetItemNumberMethod)NULL, + (GetItemIndexMethod)NULL, +}; + +} // namespace dependencies + +PyObject* NewFileDependencies(const FileDescriptor* descriptor) { + return descriptor::NewSequence(&dependencies::ContainerDef, descriptor); +} + +namespace public_dependencies { + +typedef const FileDescriptor* ItemDescriptor; + +static int Count(PyContainer* self) { + return GetDescriptor(self)->public_dependency_count(); +} + +static ItemDescriptor GetByIndex(PyContainer* self, int index) { + return GetDescriptor(self)->public_dependency(index); +} + +static PyObject* NewObjectFromItem(ItemDescriptor item) { + return PyFileDescriptor_New(item); +} + +static DescriptorContainerDef ContainerDef = { + "FilePublicDependencies", + (CountMethod)Count, + (GetByIndexMethod)GetByIndex, + (GetByNameMethod)NULL, + (GetByNumberMethod)NULL, + (NewObjectFromItemMethod)NewObjectFromItem, + (GetItemNameMethod)NULL, + (GetItemNumberMethod)NULL, + (GetItemIndexMethod)NULL, +}; + +} // namespace public_dependencies + +PyObject* NewFilePublicDependencies(const FileDescriptor* descriptor) { + return descriptor::NewSequence(&public_dependencies::ContainerDef, + descriptor); +} + +} // namespace file_descriptor + + +// Register all implementations + +bool InitDescriptorMappingTypes() { + if (PyType_Ready(&descriptor::DescriptorMapping_Type) < 0) + return false; + if (PyType_Ready(&descriptor::DescriptorSequence_Type) < 0) + return false; + if (PyType_Ready(&descriptor::ContainerIterator_Type) < 0) + return false; + return true; +} + +} // namespace python +} // namespace protobuf +} // namespace google diff --git a/python/google/protobuf/pyext/descriptor_containers.h b/python/google/protobuf/pyext/descriptor_containers.h new file mode 100644 index 00000000..d81537de --- /dev/null +++ b/python/google/protobuf/pyext/descriptor_containers.h @@ -0,0 +1,95 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Mappings and Sequences of descriptors. +// They implement containers like fields_by_name, EnumDescriptor.values... +// See descriptor_containers.cc for more description. +#include <Python.h> + +namespace google { +namespace protobuf { + +class Descriptor; +class FileDescriptor; +class EnumDescriptor; +class OneofDescriptor; + +namespace python { + +// Initialize the various types and objects. +bool InitDescriptorMappingTypes(); + +// Each function below returns a Mapping, or a Sequence of descriptors. +// They all return a new reference. + +namespace message_descriptor { +PyObject* NewMessageFieldsByName(const Descriptor* descriptor); +PyObject* NewMessageFieldsByNumber(const Descriptor* descriptor); +PyObject* NewMessageFieldsSeq(const Descriptor* descriptor); + +PyObject* NewMessageNestedTypesSeq(const Descriptor* descriptor); +PyObject* NewMessageNestedTypesByName(const Descriptor* descriptor); + +PyObject* NewMessageEnumsByName(const Descriptor* descriptor); +PyObject* NewMessageEnumsSeq(const Descriptor* descriptor); +PyObject* NewMessageEnumValuesByName(const Descriptor* descriptor); + +PyObject* NewMessageExtensionsByName(const Descriptor* descriptor); +PyObject* NewMessageExtensionsSeq(const Descriptor* descriptor); + +PyObject* NewMessageOneofsByName(const Descriptor* descriptor); +PyObject* NewMessageOneofsSeq(const Descriptor* descriptor); +} // namespace message_descriptor + +namespace enum_descriptor { +PyObject* NewEnumValuesByName(const EnumDescriptor* descriptor); +PyObject* NewEnumValuesByNumber(const EnumDescriptor* descriptor); +PyObject* NewEnumValuesSeq(const EnumDescriptor* descriptor); +} // namespace enum_descriptor + +namespace oneof_descriptor { +PyObject* NewOneofFieldsSeq(const OneofDescriptor* descriptor); +} // namespace oneof_descriptor + +namespace file_descriptor { +PyObject* NewFileMessageTypesByName(const FileDescriptor* descriptor); + +PyObject* NewFileEnumTypesByName(const FileDescriptor* descriptor); + +PyObject* NewFileExtensionsByName(const FileDescriptor* descriptor); + +PyObject* NewFileDependencies(const FileDescriptor* descriptor); +PyObject* NewFilePublicDependencies(const FileDescriptor* descriptor); +} // namespace file_descriptor + + +} // namespace python +} // namespace protobuf +} // namespace google diff --git a/python/google/protobuf/pyext/descriptor_cpp2_test.py b/python/google/protobuf/pyext/descriptor_cpp2_test.py deleted file mode 100644 index 3cf45a22..00000000 --- a/python/google/protobuf/pyext/descriptor_cpp2_test.py +++ /dev/null @@ -1,58 +0,0 @@ -#! /usr/bin/python -# -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Tests for google.protobuf.pyext behavior.""" - -__author__ = 'anuraag@google.com (Anuraag Agrawal)' - -import os -os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp' -os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION'] = '2' - -# We must set the implementation version above before the google3 imports. -# pylint: disable=g-import-not-at-top -from google.apputils import basetest -from google.protobuf.internal import api_implementation -# Run all tests from the original module by putting them in our namespace. -# pylint: disable=wildcard-import -from google.protobuf.internal.descriptor_test import * - - -class ConfirmCppApi2Test(basetest.TestCase): - - def testImplementationSetting(self): - self.assertEqual('cpp', api_implementation.Type()) - self.assertEqual(2, api_implementation.Version()) - - -if __name__ == '__main__': - basetest.main() diff --git a/python/google/protobuf/pyext/descriptor_pool.cc b/python/google/protobuf/pyext/descriptor_pool.cc new file mode 100644 index 00000000..bc3077bc --- /dev/null +++ b/python/google/protobuf/pyext/descriptor_pool.cc @@ -0,0 +1,370 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Implements the DescriptorPool, which collects all descriptors. + +#include <Python.h> + +#include <google/protobuf/descriptor.pb.h> +#include <google/protobuf/pyext/descriptor_pool.h> +#include <google/protobuf/pyext/descriptor.h> +#include <google/protobuf/pyext/scoped_pyobject_ptr.h> + +#define C(str) const_cast<char*>(str) + +#if PY_MAJOR_VERSION >= 3 + #define PyString_FromStringAndSize PyUnicode_FromStringAndSize + #if PY_VERSION_HEX < 0x03030000 + #error "Python 3.0 - 3.2 are not supported." + #endif + #define PyString_AsStringAndSize(ob, charpp, sizep) \ + (PyUnicode_Check(ob)? \ + ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \ + PyBytes_AsStringAndSize(ob, (charpp), (sizep))) +#endif + +namespace google { +namespace protobuf { +namespace python { + +namespace cdescriptor_pool { + +PyDescriptorPool* NewDescriptorPool() { + PyDescriptorPool* cdescriptor_pool = PyObject_New( + PyDescriptorPool, &PyDescriptorPool_Type); + if (cdescriptor_pool == NULL) { + return NULL; + } + + // Build a DescriptorPool for messages only declared in Python libraries. + // generated_pool() contains all messages linked in C++ libraries, and is used + // as underlay. + cdescriptor_pool->pool = new DescriptorPool(DescriptorPool::generated_pool()); + + // TODO(amauryfa): Rewrite the SymbolDatabase in C so that it uses the same + // storage. + cdescriptor_pool->classes_by_descriptor = + new PyDescriptorPool::ClassesByMessageMap(); + cdescriptor_pool->interned_descriptors = + new hash_map<const void*, PyObject *>(); + cdescriptor_pool->descriptor_options = + new hash_map<const void*, PyObject *>(); + + return cdescriptor_pool; +} + +static void Dealloc(PyDescriptorPool* self) { + typedef PyDescriptorPool::ClassesByMessageMap::iterator iterator; + for (iterator it = self->classes_by_descriptor->begin(); + it != self->classes_by_descriptor->end(); ++it) { + Py_DECREF(it->second); + } + delete self->classes_by_descriptor; + delete self->interned_descriptors; // its references were borrowed. + for (hash_map<const void*, PyObject*>::iterator it = + self->descriptor_options->begin(); + it != self->descriptor_options->end(); ++it) { + Py_DECREF(it->second); + } + delete self->descriptor_options; + Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self)); +} + +PyObject* FindMessageByName(PyDescriptorPool* self, PyObject* arg) { + Py_ssize_t name_size; + char* name; + if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) { + return NULL; + } + + const Descriptor* message_descriptor = + self->pool->FindMessageTypeByName(string(name, name_size)); + + if (message_descriptor == NULL) { + PyErr_Format(PyExc_TypeError, "Couldn't find message %.200s", name); + return NULL; + } + + return PyMessageDescriptor_New(message_descriptor); +} + +// Add a message class to our database. +const Descriptor* RegisterMessageClass( + PyDescriptorPool* self, PyObject *message_class, PyObject* descriptor) { + ScopedPyObjectPtr full_message_name( + PyObject_GetAttrString(descriptor, "full_name")); + Py_ssize_t name_size; + char* name; + if (PyString_AsStringAndSize(full_message_name, &name, &name_size) < 0) { + return NULL; + } + const Descriptor *message_descriptor = + self->pool->FindMessageTypeByName(string(name, name_size)); + if (!message_descriptor) { + PyErr_Format(PyExc_TypeError, "Could not find C++ descriptor for '%s'", + name); + return NULL; + } + Py_INCREF(message_class); + typedef PyDescriptorPool::ClassesByMessageMap::iterator iterator; + std::pair<iterator, bool> ret = self->classes_by_descriptor->insert( + std::make_pair(message_descriptor, message_class)); + if (!ret.second) { + // Update case: DECREF the previous value. + Py_DECREF(ret.first->second); + ret.first->second = message_class; + } + return message_descriptor; +} + +// Retrieve the message class added to our database. +PyObject *GetMessageClass(PyDescriptorPool* self, + const Descriptor *message_descriptor) { + typedef PyDescriptorPool::ClassesByMessageMap::iterator iterator; + iterator ret = self->classes_by_descriptor->find(message_descriptor); + if (ret == self->classes_by_descriptor->end()) { + PyErr_Format(PyExc_TypeError, "No message class registered for '%s'", + message_descriptor->full_name().c_str()); + return NULL; + } else { + return ret->second; + } +} + +PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* arg) { + Py_ssize_t name_size; + char* name; + if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) { + return NULL; + } + + const FieldDescriptor* field_descriptor = + self->pool->FindFieldByName(string(name, name_size)); + if (field_descriptor == NULL) { + PyErr_Format(PyExc_TypeError, "Couldn't find field %.200s", + name); + return NULL; + } + + return PyFieldDescriptor_New(field_descriptor); +} + +PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg) { + Py_ssize_t name_size; + char* name; + if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) { + return NULL; + } + + const FieldDescriptor* field_descriptor = + self->pool->FindExtensionByName(string(name, name_size)); + if (field_descriptor == NULL) { + PyErr_Format(PyExc_TypeError, "Couldn't find field %.200s", name); + return NULL; + } + + return PyFieldDescriptor_New(field_descriptor); +} + +PyObject* FindEnumTypeByName(PyDescriptorPool* self, PyObject* arg) { + Py_ssize_t name_size; + char* name; + if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) { + return NULL; + } + + const EnumDescriptor* enum_descriptor = + self->pool->FindEnumTypeByName(string(name, name_size)); + if (enum_descriptor == NULL) { + PyErr_Format(PyExc_TypeError, "Couldn't find enum %.200s", name); + return NULL; + } + + return PyEnumDescriptor_New(enum_descriptor); +} + +PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg) { + Py_ssize_t name_size; + char* name; + if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) { + return NULL; + } + + const OneofDescriptor* oneof_descriptor = + self->pool->FindOneofByName(string(name, name_size)); + if (oneof_descriptor == NULL) { + PyErr_Format(PyExc_TypeError, "Couldn't find oneof %.200s", name); + return NULL; + } + + return PyOneofDescriptor_New(oneof_descriptor); +} + +static PyMethodDef Methods[] = { + { C("FindFieldByName"), + (PyCFunction)FindFieldByName, + METH_O, + C("Searches for a field descriptor by full name.") }, + { C("FindExtensionByName"), + (PyCFunction)FindExtensionByName, + METH_O, + C("Searches for extension descriptor by full name.") }, + {NULL} +}; + +} // namespace cdescriptor_pool + +PyTypeObject PyDescriptorPool_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + C("google.protobuf.internal." + "_message.DescriptorPool"), // tp_name + sizeof(PyDescriptorPool), // tp_basicsize + 0, // tp_itemsize + (destructor)cdescriptor_pool::Dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + C("A Descriptor Pool"), // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + cdescriptor_pool::Methods, // tp_methods + 0, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init + 0, // tp_alloc + 0, // tp_new + PyObject_Del, // tp_free +}; + +// The code below loads new Descriptors from a serialized FileDescriptorProto. + + +// Collects errors that occur during proto file building to allow them to be +// propagated in the python exception instead of only living in ERROR logs. +class BuildFileErrorCollector : public DescriptorPool::ErrorCollector { + public: + BuildFileErrorCollector() : error_message(""), had_errors(false) {} + + void AddError(const string& filename, const string& element_name, + const Message* descriptor, ErrorLocation location, + const string& message) { + // Replicates the logging behavior that happens in the C++ implementation + // when an error collector is not passed in. + if (!had_errors) { + error_message += + ("Invalid proto descriptor for file \"" + filename + "\":\n"); + } + // As this only happens on failure and will result in the program not + // running at all, no effort is made to optimize this string manipulation. + error_message += (" " + element_name + ": " + message + "\n"); + } + + string error_message; + bool had_errors; +}; + +PyObject* Python_BuildFile(PyObject* ignored, PyObject* serialized_pb) { + char* message_type; + Py_ssize_t message_len; + + if (PyBytes_AsStringAndSize(serialized_pb, &message_type, &message_len) < 0) { + return NULL; + } + + FileDescriptorProto file_proto; + if (!file_proto.ParseFromArray(message_type, message_len)) { + PyErr_SetString(PyExc_TypeError, "Couldn't parse file content!"); + return NULL; + } + + // If the file was already part of a C++ library, all its descriptors are in + // the underlying pool. No need to do anything else. + const FileDescriptor* generated_file = + DescriptorPool::generated_pool()->FindFileByName(file_proto.name()); + if (generated_file != NULL) { + return PyFileDescriptor_NewWithPb(generated_file, serialized_pb); + } + + BuildFileErrorCollector error_collector; + const FileDescriptor* descriptor = + GetDescriptorPool()->pool->BuildFileCollectingErrors(file_proto, + &error_collector); + if (descriptor == NULL) { + PyErr_Format(PyExc_TypeError, + "Couldn't build proto file into descriptor pool!\n%s", + error_collector.error_message.c_str()); + return NULL; + } + + return PyFileDescriptor_NewWithPb(descriptor, serialized_pb); +} + +static PyDescriptorPool* global_cdescriptor_pool = NULL; + +bool InitDescriptorPool() { + if (PyType_Ready(&PyDescriptorPool_Type) < 0) + return false; + + global_cdescriptor_pool = cdescriptor_pool::NewDescriptorPool(); + if (global_cdescriptor_pool == NULL) { + return false; + } + + return true; +} + +PyDescriptorPool* GetDescriptorPool() { + return global_cdescriptor_pool; +} + +} // namespace python +} // namespace protobuf +} // namespace google diff --git a/python/google/protobuf/pyext/descriptor_pool.h b/python/google/protobuf/pyext/descriptor_pool.h new file mode 100644 index 00000000..4e494b89 --- /dev/null +++ b/python/google/protobuf/pyext/descriptor_pool.h @@ -0,0 +1,152 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_POOL_H__ +#define GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_POOL_H__ + +#include <Python.h> + +#include <google/protobuf/stubs/hash.h> +#include <google/protobuf/descriptor.h> + +namespace google { +namespace protobuf { +namespace python { + +// Wraps operations to the global DescriptorPool which contains information +// about all messages and fields. +// +// There is normally one pool per process. We make it a Python object only +// because it contains many Python references. +// TODO(amauryfa): See whether such objects can appear in reference cycles, and +// consider adding support for the cyclic GC. +// +// "Methods" that interacts with this DescriptorPool are in the cdescriptor_pool +// namespace. +typedef struct PyDescriptorPool { + PyObject_HEAD + + DescriptorPool* pool; + + // Make our own mapping to retrieve Python classes from C++ descriptors. + // + // Descriptor pointers stored here are owned by the DescriptorPool above. + // Python references to classes are owned by this PyDescriptorPool. + typedef hash_map<const Descriptor*, PyObject*> ClassesByMessageMap; + ClassesByMessageMap* classes_by_descriptor; + + // Store interned descriptors, so that the same C++ descriptor yields the same + // Python object. Objects are not immortal: this map does not own the + // references, and items are deleted when the last reference to the object is + // released. + // This is enough to support the "is" operator on live objects. + // All descriptors are stored here. + hash_map<const void*, PyObject*>* interned_descriptors; + + // Cache the options for any kind of descriptor. + // Descriptor pointers are owned by the DescriptorPool above. + // Python objects are owned by the map. + hash_map<const void*, PyObject*>* descriptor_options; +} PyDescriptorPool; + + +extern PyTypeObject PyDescriptorPool_Type; + +namespace cdescriptor_pool { + +// Builds a new DescriptorPool. Normally called only once per process. +PyDescriptorPool* NewDescriptorPool(); + +// Looks up a message by name. +// Returns a message Descriptor, or NULL if not found. +const Descriptor* FindMessageTypeByName(PyDescriptorPool* self, + const string& name); + +// Registers a new Python class for the given message descriptor. +// Returns the message Descriptor. +// On error, returns NULL with a Python exception set. +const Descriptor* RegisterMessageClass( + PyDescriptorPool* self, PyObject* message_class, PyObject* descriptor); + +// Retrieves the Python class registered with the given message descriptor. +// +// Returns a *borrowed* reference if found, otherwise returns NULL with an +// exception set. +PyObject* GetMessageClass(PyDescriptorPool* self, + const Descriptor* message_descriptor); + +// Looks up a message by name. Returns a PyMessageDescriptor corresponding to +// the field on success, or NULL on failure. +// +// Returns a new reference. +PyObject* FindMessageByName(PyDescriptorPool* self, PyObject* name); + +// Looks up a field by name. Returns a PyFieldDescriptor corresponding to +// the field on success, or NULL on failure. +// +// Returns a new reference. +PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* name); + +// Looks up an extension by name. Returns a PyFieldDescriptor corresponding +// to the field on success, or NULL on failure. +// +// Returns a new reference. +PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg); + +// Looks up an enum type by name. Returns a PyEnumDescriptor corresponding +// to the field on success, or NULL on failure. +// +// Returns a new reference. +PyObject* FindEnumTypeByName(PyDescriptorPool* self, PyObject* arg); + +// Looks up a oneof by name. Returns a COneofDescriptor corresponding +// to the oneof on success, or NULL on failure. +// +// Returns a new reference. +PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg); + +} // namespace cdescriptor_pool + +// Implement the Python "_BuildFile" method, it takes a serialized +// FileDescriptorProto, and adds it to the C++ DescriptorPool. +// It returns a new FileDescriptor object, or NULL when an exception is raised. +PyObject* Python_BuildFile(PyObject* ignored, PyObject* args); + +// Retrieve the global descriptor pool owned by the _message module. +PyDescriptorPool* GetDescriptorPool(); + +// Initialize objects used by this module. +bool InitDescriptorPool(); + +} // namespace python +} // namespace protobuf + +} // namespace google +#endif // GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_POOL_H__ diff --git a/python/google/protobuf/pyext/extension_dict.cc b/python/google/protobuf/pyext/extension_dict.cc index d83b57d5..8e38fc42 100644 --- a/python/google/protobuf/pyext/extension_dict.cc +++ b/python/google/protobuf/pyext/extension_dict.cc @@ -38,6 +38,7 @@ #include <google/protobuf/dynamic_message.h> #include <google/protobuf/message.h> #include <google/protobuf/pyext/descriptor.h> +#include <google/protobuf/pyext/descriptor_pool.h> #include <google/protobuf/pyext/message.h> #include <google/protobuf/pyext/repeated_composite_container.h> #include <google/protobuf/pyext/repeated_scalar_container.h> @@ -48,13 +49,11 @@ namespace google { namespace protobuf { namespace python { -extern google::protobuf::DynamicMessageFactory* global_message_factory; - namespace extension_dict { // TODO(tibell): Always use self->message for clarity, just like in // RepeatedCompositeContainer. -static google::protobuf::Message* GetMessage(ExtensionDict* self) { +static Message* GetMessage(ExtensionDict* self) { if (self->parent != NULL) { return self->parent->message; } else { @@ -73,10 +72,9 @@ PyObject* len(ExtensionDict* self) { // TODO(tibell): Use VisitCompositeField. int ReleaseExtension(ExtensionDict* self, PyObject* extension, - const google::protobuf::FieldDescriptor* descriptor) { - if (descriptor->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) { - if (descriptor->cpp_type() == - google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + const FieldDescriptor* descriptor) { + if (descriptor->label() == FieldDescriptor::LABEL_REPEATED) { + if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { if (repeated_composite_container::Release( reinterpret_cast<RepeatedCompositeContainer*>( extension)) < 0) { @@ -89,8 +87,7 @@ int ReleaseExtension(ExtensionDict* self, return -1; } } - } else if (descriptor->cpp_type() == - google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + } else if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { if (cmessage::ReleaseSubMessage( GetMessage(self), descriptor, reinterpret_cast<CMessage*>(extension)) < 0) { @@ -102,8 +99,7 @@ int ReleaseExtension(ExtensionDict* self, } PyObject* subscript(ExtensionDict* self, PyObject* key) { - const google::protobuf::FieldDescriptor* descriptor = - cmessage::GetExtensionDescriptor(key); + const FieldDescriptor* descriptor = cmessage::GetExtensionDescriptor(key); if (descriptor == NULL) { return NULL; } @@ -162,8 +158,7 @@ PyObject* subscript(ExtensionDict* self, PyObject* key) { } int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value) { - const google::protobuf::FieldDescriptor* descriptor = - cmessage::GetExtensionDescriptor(key); + const FieldDescriptor* descriptor = cmessage::GetExtensionDescriptor(key); if (descriptor == NULL) { return -1; } @@ -187,7 +182,7 @@ int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value) { } PyObject* ClearExtension(ExtensionDict* self, PyObject* extension) { - const google::protobuf::FieldDescriptor* descriptor = + const FieldDescriptor* descriptor = cmessage::GetExtensionDescriptor(extension); if (descriptor == NULL) { return NULL; @@ -208,7 +203,7 @@ PyObject* ClearExtension(ExtensionDict* self, PyObject* extension) { } PyObject* HasExtension(ExtensionDict* self, PyObject* extension) { - const google::protobuf::FieldDescriptor* descriptor = + const FieldDescriptor* descriptor = cmessage::GetExtensionDescriptor(extension); if (descriptor == NULL) { return NULL; diff --git a/python/google/protobuf/pyext/extension_dict.h b/python/google/protobuf/pyext/extension_dict.h index 47625e23..7e1049f1 100644 --- a/python/google/protobuf/pyext/extension_dict.h +++ b/python/google/protobuf/pyext/extension_dict.h @@ -41,7 +41,6 @@ #include <google/protobuf/stubs/shared_ptr.h> #endif - namespace google { namespace protobuf { @@ -94,7 +93,7 @@ PyObject* len(ExtensionDict* self); // Returns 0 on success, -1 on failure. int ReleaseExtension(ExtensionDict* self, PyObject* extension, - const google::protobuf::FieldDescriptor* descriptor); + const FieldDescriptor* descriptor); // Gets an extension from the dict for the given extension descriptor. // diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc index cd956e0e..c48f94c3 100644 --- a/python/google/protobuf/pyext/message.cc +++ b/python/google/protobuf/pyext/message.cc @@ -39,6 +39,7 @@ #endif #include <string> #include <vector> +#include <structmember.h> // A Python header file. #ifndef PyVarObject_HEAD_INIT #define PyVarObject_HEAD_INIT(type, size) PyObject_HEAD_INIT(type) size, @@ -54,10 +55,12 @@ #include <google/protobuf/message.h> #include <google/protobuf/text_format.h> #include <google/protobuf/pyext/descriptor.h> +#include <google/protobuf/pyext/descriptor_pool.h> #include <google/protobuf/pyext/extension_dict.h> #include <google/protobuf/pyext/repeated_composite_container.h> #include <google/protobuf/pyext/repeated_scalar_container.h> #include <google/protobuf/pyext/scoped_pyobject_ptr.h> +#include <google/protobuf/stubs/strutil.h> #if PY_MAJOR_VERSION >= 3 #define PyInt_Check PyLong_Check @@ -72,6 +75,10 @@ #else #define PyString_AsString(ob) \ (PyUnicode_Check(ob)? PyUnicode_AsUTF8(ob): PyBytes_AsString(ob)) + #define PyString_AsStringAndSize(ob, charpp, sizep) \ + (PyUnicode_Check(ob)? \ + ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \ + PyBytes_AsStringAndSize(ob, (charpp), (sizep))) #endif #endif @@ -81,14 +88,14 @@ namespace python { // Forward declarations namespace cmessage { -static const google::protobuf::FieldDescriptor* GetFieldDescriptor( +static const FieldDescriptor* GetFieldDescriptor( CMessage* self, PyObject* name); -static const google::protobuf::Descriptor* GetMessageDescriptor(PyTypeObject* cls); +static const Descriptor* GetMessageDescriptor(PyTypeObject* cls); static string GetMessageName(CMessage* self); int InternalReleaseFieldByDescriptor( - const google::protobuf::FieldDescriptor* field_descriptor, + const FieldDescriptor* field_descriptor, PyObject* composite_field, - google::protobuf::Message* parent_message); + Message* parent_message); } // namespace cmessage // --------------------------------------------------------------------- @@ -107,7 +114,7 @@ struct ChildVisitor { // Returns 0 on success, -1 on failure. int VisitCMessage(CMessage* cmessage, - const google::protobuf::FieldDescriptor* field_descriptor) { + const FieldDescriptor* field_descriptor) { return 0; } }; @@ -118,8 +125,8 @@ template<class Visitor> static int VisitCompositeField(const FieldDescriptor* descriptor, PyObject* child, Visitor visitor) { - if (descriptor->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) { - if (descriptor->cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + if (descriptor->label() == FieldDescriptor::LABEL_REPEATED) { + if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { RepeatedCompositeContainer* container = reinterpret_cast<RepeatedCompositeContainer*>(child); if (visitor.VisitRepeatedCompositeContainer(container) == -1) @@ -130,8 +137,7 @@ static int VisitCompositeField(const FieldDescriptor* descriptor, if (visitor.VisitRepeatedScalarContainer(container) == -1) return -1; } - } else if (descriptor->cpp_type() == - google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + } else if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { CMessage* cmsg = reinterpret_cast<CMessage*>(child); if (visitor.VisitCMessage(cmsg, descriptor) == -1) return -1; @@ -149,25 +155,31 @@ int ForEachCompositeField(CMessage* self, Visitor visitor) { PyObject* key; PyObject* field; - // Never use self->message in this function, it may be already freed. - const google::protobuf::Descriptor* message_descriptor = - cmessage::GetMessageDescriptor(Py_TYPE(self)); - // Visit normal fields. - while (PyDict_Next(self->composite_fields, &pos, &key, &field)) { - const google::protobuf::FieldDescriptor* descriptor = - message_descriptor->FindFieldByName(PyString_AsString(key)); - if (descriptor != NULL) { - if (VisitCompositeField(descriptor, field, visitor) == -1) + if (self->composite_fields) { + // Never use self->message in this function, it may be already freed. + const Descriptor* message_descriptor = + cmessage::GetMessageDescriptor(Py_TYPE(self)); + while (PyDict_Next(self->composite_fields, &pos, &key, &field)) { + Py_ssize_t key_str_size; + char *key_str_data; + if (PyString_AsStringAndSize(key, &key_str_data, &key_str_size) != 0) return -1; + const string key_str(key_str_data, key_str_size); + const FieldDescriptor* descriptor = + message_descriptor->FindFieldByName(key_str); + if (descriptor != NULL) { + if (VisitCompositeField(descriptor, field, visitor) == -1) + return -1; + } } } // Visit extension fields. if (self->extensions != NULL) { + pos = 0; while (PyDict_Next(self->extensions->values, &pos, &key, &field)) { - const google::protobuf::FieldDescriptor* descriptor = - cmessage::GetExtensionDescriptor(key); + const FieldDescriptor* descriptor = cmessage::GetExtensionDescriptor(key); if (descriptor == NULL) return -1; if (VisitCompositeField(descriptor, field, visitor) == -1) @@ -180,6 +192,8 @@ int ForEachCompositeField(CMessage* self, Visitor visitor) { // --------------------------------------------------------------------- +static DynamicMessageFactory* message_factory; + // Constants used for integer type range checking. PyObject* kPythonZero; PyObject* kint32min_py; @@ -198,17 +212,8 @@ PyObject* PickleError_class; static PyObject* kDESCRIPTOR; static PyObject* k_cdescriptor; static PyObject* kfull_name; -static PyObject* kname; -static PyObject* kextensions_by_name; static PyObject* k_extensions_by_name; static PyObject* k_extensions_by_number; -static PyObject* kfields_by_name; - -static PyDescriptorPool* descriptor_pool; - -PyDescriptorPool* GetDescriptorPool() { - return descriptor_pool; -} /* Is 64bit */ void FormatTypeError(PyObject* arg, char* expected_types) { @@ -305,14 +310,14 @@ bool CheckAndGetBool(PyObject* arg, bool* value) { } bool CheckAndSetString( - PyObject* arg, google::protobuf::Message* message, - const google::protobuf::FieldDescriptor* descriptor, - const google::protobuf::Reflection* reflection, + PyObject* arg, Message* message, + const FieldDescriptor* descriptor, + const Reflection* reflection, bool append, int index) { - GOOGLE_DCHECK(descriptor->type() == google::protobuf::FieldDescriptor::TYPE_STRING || - descriptor->type() == google::protobuf::FieldDescriptor::TYPE_BYTES); - if (descriptor->type() == google::protobuf::FieldDescriptor::TYPE_STRING) { + GOOGLE_DCHECK(descriptor->type() == FieldDescriptor::TYPE_STRING || + descriptor->type() == FieldDescriptor::TYPE_BYTES); + if (descriptor->type() == FieldDescriptor::TYPE_STRING) { if (!PyBytes_Check(arg) && !PyUnicode_Check(arg)) { FormatTypeError(arg, "bytes, unicode"); return false; @@ -339,7 +344,7 @@ bool CheckAndSetString( } PyObject* encoded_string = NULL; - if (descriptor->type() == google::protobuf::FieldDescriptor::TYPE_STRING) { + if (descriptor->type() == FieldDescriptor::TYPE_STRING) { if (PyBytes_Check(arg)) { // The bytes were already validated as correctly encoded UTF-8 above. encoded_string = arg; // Already encoded. @@ -376,9 +381,8 @@ bool CheckAndSetString( return true; } -PyObject* ToStringObject( - const google::protobuf::FieldDescriptor* descriptor, string value) { - if (descriptor->type() != google::protobuf::FieldDescriptor::TYPE_STRING) { +PyObject* ToStringObject(const FieldDescriptor* descriptor, string value) { + if (descriptor->type() != FieldDescriptor::TYPE_STRING) { return PyBytes_FromStringAndSize(value.c_str(), value.length()); } @@ -394,8 +398,8 @@ PyObject* ToStringObject( return result; } -bool CheckFieldBelongsToMessage(const google::protobuf::FieldDescriptor* field_descriptor, - const google::protobuf::Message* message) { +bool CheckFieldBelongsToMessage(const FieldDescriptor* field_descriptor, + const Message* message) { if (message->GetDescriptor() == field_descriptor->containing_type()) { return true; } @@ -405,16 +409,18 @@ bool CheckFieldBelongsToMessage(const google::protobuf::FieldDescriptor* field_d return false; } -google::protobuf::DynamicMessageFactory* global_message_factory; - namespace cmessage { +DynamicMessageFactory* GetMessageFactory() { + return message_factory; +} + static int MaybeReleaseOverlappingOneofField( CMessage* cmessage, - const google::protobuf::FieldDescriptor* field) { + const FieldDescriptor* field) { #ifdef GOOGLE_PROTOBUF_HAS_ONEOF - google::protobuf::Message* message = cmessage->message; - const google::protobuf::Reflection* reflection = message->GetReflection(); + Message* message = cmessage->message; + const Reflection* reflection = message->GetReflection(); if (!field->containing_oneof() || !reflection->HasOneof(*message, field->containing_oneof()) || reflection->HasField(*message, field)) { @@ -425,13 +431,13 @@ static int MaybeReleaseOverlappingOneofField( const OneofDescriptor* oneof = field->containing_oneof(); const FieldDescriptor* existing_field = reflection->GetOneofFieldDescriptor(*message, oneof); - if (existing_field->cpp_type() != google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + if (existing_field->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) { // Non-message fields don't need to be released. return 0; } const char* field_name = existing_field->name().c_str(); - PyObject* child_message = PyDict_GetItemString( - cmessage->composite_fields, field_name); + PyObject* child_message = cmessage->composite_fields ? + PyDict_GetItemString(cmessage->composite_fields, field_name) : NULL; if (child_message == NULL) { // No python reference to this field so no need to release. return 0; @@ -450,21 +456,21 @@ static int MaybeReleaseOverlappingOneofField( // --------------------------------------------------------------------- // Making a message writable -static google::protobuf::Message* GetMutableMessage( +static Message* GetMutableMessage( CMessage* parent, - const google::protobuf::FieldDescriptor* parent_field) { - google::protobuf::Message* parent_message = parent->message; - const google::protobuf::Reflection* reflection = parent_message->GetReflection(); + const FieldDescriptor* parent_field) { + Message* parent_message = parent->message; + const Reflection* reflection = parent_message->GetReflection(); if (MaybeReleaseOverlappingOneofField(parent, parent_field) < 0) { return NULL; } return reflection->MutableMessage( - parent_message, parent_field, global_message_factory); + parent_message, parent_field, message_factory); } struct FixupMessageReference : public ChildVisitor { // message must outlive this object. - explicit FixupMessageReference(google::protobuf::Message* message) : + explicit FixupMessageReference(Message* message) : message_(message) {} int VisitRepeatedCompositeContainer(RepeatedCompositeContainer* container) { @@ -478,7 +484,7 @@ struct FixupMessageReference : public ChildVisitor { } private: - google::protobuf::Message* message_; + Message* message_; }; int AssureWritable(CMessage* self) { @@ -490,7 +496,7 @@ int AssureWritable(CMessage* self) { // If parent is NULL but we are trying to modify a read-only message, this // is a reference to a constant default instance that needs to be replaced // with a mutable top-level message. - const Message* prototype = global_message_factory->GetPrototype( + const Message* prototype = message_factory->GetPrototype( self->message->GetDescriptor()); self->message = prototype->New(); self->owner.reset(self->message); @@ -500,8 +506,8 @@ int AssureWritable(CMessage* self) { return -1; // Make self->message writable. - google::protobuf::Message* parent_message = self->parent->message; - google::protobuf::Message* mutable_message = GetMutableMessage( + Message* parent_message = self->parent->message; + Message* mutable_message = GetMutableMessage( self->parent, self->parent_field_descriptor); if (mutable_message == NULL) { @@ -528,38 +534,35 @@ int AssureWritable(CMessage* self) { // Retrieve the C++ Descriptor of a message class. // On error, returns NULL with an exception set. -static const google::protobuf::Descriptor* GetMessageDescriptor(PyTypeObject* cls) { +static const Descriptor* GetMessageDescriptor(PyTypeObject* cls) { ScopedPyObjectPtr descriptor(PyObject_GetAttr( reinterpret_cast<PyObject*>(cls), kDESCRIPTOR)); if (descriptor == NULL) { PyErr_SetString(PyExc_TypeError, "Message class has no DESCRIPTOR"); return NULL; } - ScopedPyObjectPtr cdescriptor(PyObject_GetAttr(descriptor, k_cdescriptor)); - if (cdescriptor == NULL) { - PyErr_SetString(PyExc_TypeError, "Unregistered message."); + if (!PyObject_TypeCheck(descriptor, &PyMessageDescriptor_Type)) { + PyErr_Format(PyExc_TypeError, "Expected a message Descriptor, got %s", + descriptor->ob_type->tp_name); return NULL; } - if (!PyObject_TypeCheck(cdescriptor, &CMessageDescriptor_Type)) { - PyErr_SetString(PyExc_TypeError, "Not a CMessageDescriptor"); - return NULL; - } - return reinterpret_cast<CMessageDescriptor*>(cdescriptor.get())->descriptor; + return PyMessageDescriptor_AsDescriptor(descriptor); } // Retrieve a C++ FieldDescriptor for a message attribute. // The C++ message must be valid. // TODO(amauryfa): This function should stay internal, because exception // handling is not consistent. -static const google::protobuf::FieldDescriptor* GetFieldDescriptor( +static const FieldDescriptor* GetFieldDescriptor( CMessage* self, PyObject* name) { - const google::protobuf::Descriptor *message_descriptor = self->message->GetDescriptor(); - const char* field_name = PyString_AsString(name); - if (field_name == NULL) { + const Descriptor *message_descriptor = self->message->GetDescriptor(); + char* field_name; + Py_ssize_t size; + if (PyString_AsStringAndSize(name, &field_name, &size) < 0) { return NULL; } - const google::protobuf::FieldDescriptor *field_descriptor = - message_descriptor->FindFieldByName(field_name); + const FieldDescriptor *field_descriptor = + message_descriptor->FindFieldByName(string(field_name, size)); if (field_descriptor == NULL) { // Note: No exception is set! return NULL; @@ -568,19 +571,16 @@ static const google::protobuf::FieldDescriptor* GetFieldDescriptor( } // Retrieve a C++ FieldDescriptor for an extension handle. -const google::protobuf::FieldDescriptor* GetExtensionDescriptor(PyObject* extension) { - ScopedPyObjectPtr cdescriptor( - PyObject_GetAttrString(extension, "_cdescriptor")); - if (cdescriptor == NULL) { - PyErr_SetString(PyExc_KeyError, "Unregistered extension."); +const FieldDescriptor* GetExtensionDescriptor(PyObject* extension) { + ScopedPyObjectPtr cdescriptor; + if (!PyObject_TypeCheck(extension, &PyFieldDescriptor_Type)) { + // Most callers consider extensions as a plain dictionary. We should + // allow input which is not a field descriptor, and simply pretend it does + // not exist. + PyErr_SetObject(PyExc_KeyError, extension); return NULL; } - if (!PyObject_TypeCheck(cdescriptor, &CFieldDescriptor_Type)) { - PyErr_SetString(PyExc_TypeError, "Not a CFieldDescriptor"); - Py_DECREF(cdescriptor); - return NULL; - } - return reinterpret_cast<CFieldDescriptor*>(cdescriptor.get())->descriptor; + return PyFieldDescriptor_AsDescriptor(extension); } // If cmessage_list is not NULL, this function releases values into the @@ -588,12 +588,12 @@ const google::protobuf::FieldDescriptor* GetExtensionDescriptor(PyObject* extens // needs to do this to make sure CMessages stay alive if they're still // referenced after deletion. Repeated scalar container doesn't need to worry. int InternalDeleteRepeatedField( - google::protobuf::Message* message, - const google::protobuf::FieldDescriptor* field_descriptor, + Message* message, + const FieldDescriptor* field_descriptor, PyObject* slice, PyObject* cmessage_list) { Py_ssize_t length, from, to, step, slice_length; - const google::protobuf::Reflection* reflection = message->GetReflection(); + const Reflection* reflection = message->GetReflection(); int min, max; length = reflection->FieldSize(*message, field_descriptor); @@ -690,18 +690,18 @@ int InitAttributes(CMessage* self, PyObject* kwargs) { PyErr_SetString(PyExc_ValueError, "Field name must be a string"); return -1; } - const google::protobuf::FieldDescriptor* descriptor = GetFieldDescriptor(self, name); + const FieldDescriptor* descriptor = GetFieldDescriptor(self, name); if (descriptor == NULL) { PyErr_Format(PyExc_ValueError, "Protocol message has no \"%s\" field.", PyString_AsString(name)); return -1; } - if (descriptor->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) { + if (descriptor->label() == FieldDescriptor::LABEL_REPEATED) { ScopedPyObjectPtr container(GetAttr(self, name)); if (container == NULL) { return -1; } - if (descriptor->cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { if (repeated_composite_container::Extend( reinterpret_cast<RepeatedCompositeContainer*>(container.get()), value) @@ -716,8 +716,7 @@ int InitAttributes(CMessage* self, PyObject* kwargs) { return -1; } } - } else if (descriptor->cpp_type() == - google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + } else if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { ScopedPyObjectPtr message(GetAttr(self, name)); if (message == NULL) { return -1; @@ -737,8 +736,7 @@ int InitAttributes(CMessage* self, PyObject* kwargs) { // Allocates an incomplete Python Message: the caller must fill self->message, // self->owner and eventually self->parent. -CMessage* NewEmptyMessage(PyObject* type, - const google::protobuf::Descriptor *descriptor) { +CMessage* NewEmptyMessage(PyObject* type, const Descriptor *descriptor) { CMessage* self = reinterpret_cast<CMessage*>( PyType_GenericAlloc(reinterpret_cast<PyTypeObject*>(type), 0)); if (self == NULL) { @@ -751,10 +749,7 @@ CMessage* NewEmptyMessage(PyObject* type, self->read_only = false; self->extensions = NULL; - self->composite_fields = PyDict_New(); - if (self->composite_fields == NULL) { - return NULL; - } + self->composite_fields = NULL; // If there are extension_ranges, the message is "extendable". Allocate a // dictionary to store the extension fields. @@ -776,12 +771,12 @@ CMessage* NewEmptyMessage(PyObject* type, static PyObject* New(PyTypeObject* type, PyObject* unused_args, PyObject* unused_kwargs) { // Retrieve the message descriptor and the default instance (=prototype). - const google::protobuf::Descriptor* message_descriptor = GetMessageDescriptor(type); + const Descriptor* message_descriptor = GetMessageDescriptor(type); if (message_descriptor == NULL) { return NULL; } - const google::protobuf::Message* default_message = - global_message_factory->GetPrototype(message_descriptor); + const Message* default_message = + message_factory->GetPrototype(message_descriptor); if (default_message == NULL) { PyErr_SetString(PyExc_TypeError, message_descriptor->full_name().c_str()); return NULL; @@ -836,7 +831,7 @@ struct ClearWeakReferences : public ChildVisitor { } int VisitCMessage(CMessage* cmessage, - const google::protobuf::FieldDescriptor* field_descriptor) { + const FieldDescriptor* field_descriptor) { cmessage->parent = NULL; return 0; } @@ -886,12 +881,12 @@ PyObject* IsInitialized(CMessage* self, PyObject* args) { } PyObject* HasFieldByDescriptor( - CMessage* self, const google::protobuf::FieldDescriptor* field_descriptor) { - google::protobuf::Message* message = self->message; + CMessage* self, const FieldDescriptor* field_descriptor) { + Message* message = self->message; if (!CheckFieldBelongsToMessage(field_descriptor, message)) { return NULL; } - if (field_descriptor->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) { + if (field_descriptor->label() == FieldDescriptor::LABEL_REPEATED) { PyErr_SetString(PyExc_KeyError, "Field is repeated. A singular method is required."); return NULL; @@ -901,42 +896,78 @@ PyObject* HasFieldByDescriptor( return PyBool_FromLong(has_field ? 1 : 0); } -const google::protobuf::FieldDescriptor* FindFieldWithOneofs( - const google::protobuf::Message* message, const char* field_name, bool* in_oneof) { - const google::protobuf::Descriptor* descriptor = message->GetDescriptor(); - const google::protobuf::FieldDescriptor* field_descriptor = +const FieldDescriptor* FindFieldWithOneofs( + const Message* message, const string& field_name, bool* in_oneof) { + *in_oneof = false; + const Descriptor* descriptor = message->GetDescriptor(); + const FieldDescriptor* field_descriptor = descriptor->FindFieldByName(field_name); - if (field_descriptor == NULL) { - const google::protobuf::OneofDescriptor* oneof_desc = - message->GetDescriptor()->FindOneofByName(field_name); - if (oneof_desc == NULL) { - *in_oneof = false; - return NULL; - } else { - *in_oneof = true; - return message->GetReflection()->GetOneofFieldDescriptor( - *message, oneof_desc); + if (field_descriptor != NULL) { + return field_descriptor; + } + const OneofDescriptor* oneof_desc = + descriptor->FindOneofByName(field_name); + if (oneof_desc != NULL) { + *in_oneof = true; + return message->GetReflection()->GetOneofFieldDescriptor(*message, + oneof_desc); + } + return NULL; +} + +bool CheckHasPresence(const FieldDescriptor* field_descriptor, bool in_oneof) { + if (field_descriptor->label() == FieldDescriptor::LABEL_REPEATED) { + PyErr_Format(PyExc_ValueError, + "Protocol message has no singular \"%s\" field.", + field_descriptor->name().c_str()); + return false; + } + + if (field_descriptor->file()->syntax() == FileDescriptor::SYNTAX_PROTO3) { + // HasField() for a oneof *itself* isn't supported. + if (in_oneof) { + PyErr_Format(PyExc_ValueError, + "Can't test oneof field \"%s\" for presence in proto3, use " + "WhichOneof instead.", + field_descriptor->containing_oneof()->name().c_str()); + return false; + } + + // ...but HasField() for fields *in* a oneof is supported. + if (field_descriptor->containing_oneof() != NULL) { + return true; + } + + if (field_descriptor->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) { + PyErr_Format( + PyExc_ValueError, + "Can't test non-submessage field \"%s\" for presence in proto3.", + field_descriptor->name().c_str()); + return false; } } - return field_descriptor; + + return true; } PyObject* HasField(CMessage* self, PyObject* arg) { -#if PY_MAJOR_VERSION < 3 char* field_name; - if (PyString_AsStringAndSize(arg, &field_name, NULL) < 0) { + Py_ssize_t size; +#if PY_MAJOR_VERSION < 3 + if (PyString_AsStringAndSize(arg, &field_name, &size) < 0) { + return NULL; + } #else - char* field_name = PyUnicode_AsUTF8(arg); + field_name = PyUnicode_AsUTF8AndSize(arg, &size); if (!field_name) { -#endif return NULL; } +#endif - google::protobuf::Message* message = self->message; - const google::protobuf::Descriptor* descriptor = message->GetDescriptor(); + Message* message = self->message; bool is_in_oneof; - const google::protobuf::FieldDescriptor* field_descriptor = - FindFieldWithOneofs(message, field_name, &is_in_oneof); + const FieldDescriptor* field_descriptor = + FindFieldWithOneofs(message, string(field_name, size), &is_in_oneof); if (field_descriptor == NULL) { if (!is_in_oneof) { PyErr_Format(PyExc_ValueError, "Unknown field %s.", field_name); @@ -946,28 +977,28 @@ PyObject* HasField(CMessage* self, PyObject* arg) { } } - if (field_descriptor->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) { - PyErr_Format(PyExc_ValueError, - "Protocol message has no singular \"%s\" field.", field_name); + if (!CheckHasPresence(field_descriptor, is_in_oneof)) { return NULL; } - bool has_field = - message->GetReflection()->HasField(*message, field_descriptor); - if (!has_field && field_descriptor->cpp_type() == - google::protobuf::FieldDescriptor::CPPTYPE_ENUM) { - // We may have an invalid enum value stored in the UnknownFieldSet and need - // to check presence in there as well. - const google::protobuf::UnknownFieldSet& unknown_field_set = + if (message->GetReflection()->HasField(*message, field_descriptor)) { + Py_RETURN_TRUE; + } + if (!message->GetReflection()->SupportsUnknownEnumValues() && + field_descriptor->cpp_type() == FieldDescriptor::CPPTYPE_ENUM) { + // Special case: Python HasField() differs in semantics from C++ + // slightly: we return HasField('enum_field') == true if there is + // an unknown enum value present. To implement this we have to + // look in the UnknownFieldSet. + const UnknownFieldSet& unknown_field_set = message->GetReflection()->GetUnknownFields(*message); for (int i = 0; i < unknown_field_set.field_count(); ++i) { if (unknown_field_set.field(i).number() == field_descriptor->number()) { Py_RETURN_TRUE; } } - Py_RETURN_FALSE; } - return PyBool_FromLong(has_field ? 1 : 0); + Py_RETURN_FALSE; } PyObject* ClearExtension(CMessage* self, PyObject* arg) { @@ -1034,7 +1065,7 @@ struct SetOwnerVisitor : public ChildVisitor { } int VisitCMessage(CMessage* cmessage, - const google::protobuf::FieldDescriptor* field_descriptor) { + const FieldDescriptor* field_descriptor) { return SetOwner(cmessage, new_owner_); } @@ -1053,18 +1084,17 @@ int SetOwner(CMessage* self, const shared_ptr<Message>& new_owner) { // Releases the message specified by 'field' and returns the // pointer. If the field does not exist a new message is created using // 'descriptor'. The caller takes ownership of the returned pointer. -Message* ReleaseMessage(google::protobuf::Message* message, - const google::protobuf::Descriptor* descriptor, - const google::protobuf::FieldDescriptor* field_descriptor) { +Message* ReleaseMessage(Message* message, + const Descriptor* descriptor, + const FieldDescriptor* field_descriptor) { Message* released_message = message->GetReflection()->ReleaseMessage( - message, field_descriptor, global_message_factory); + message, field_descriptor, message_factory); // ReleaseMessage will return NULL which differs from // child_cmessage->message, if the field does not exist. In this case, // the latter points to the default instance via a const_cast<>, so we // have to reset it to a new mutable object since we are taking ownership. if (released_message == NULL) { - const Message* prototype = global_message_factory->GetPrototype( - descriptor); + const Message* prototype = message_factory->GetPrototype(descriptor); GOOGLE_DCHECK(prototype != NULL); released_message = prototype->New(); } @@ -1072,8 +1102,8 @@ Message* ReleaseMessage(google::protobuf::Message* message, return released_message; } -int ReleaseSubMessage(google::protobuf::Message* message, - const google::protobuf::FieldDescriptor* field_descriptor, +int ReleaseSubMessage(Message* message, + const FieldDescriptor* field_descriptor, CMessage* child_cmessage) { // Release the Message shared_ptr<Message> released_message(ReleaseMessage( @@ -1089,7 +1119,7 @@ int ReleaseSubMessage(google::protobuf::Message* message, struct ReleaseChild : public ChildVisitor { // message must outlive this object. - explicit ReleaseChild(google::protobuf::Message* parent_message) : + explicit ReleaseChild(Message* parent_message) : parent_message_(parent_message) {} int VisitRepeatedCompositeContainer(RepeatedCompositeContainer* container) { @@ -1103,38 +1133,27 @@ struct ReleaseChild : public ChildVisitor { } int VisitCMessage(CMessage* cmessage, - const google::protobuf::FieldDescriptor* field_descriptor) { + const FieldDescriptor* field_descriptor) { return ReleaseSubMessage(parent_message_, field_descriptor, reinterpret_cast<CMessage*>(cmessage)); } - google::protobuf::Message* parent_message_; + Message* parent_message_; }; int InternalReleaseFieldByDescriptor( - const google::protobuf::FieldDescriptor* field_descriptor, + const FieldDescriptor* field_descriptor, PyObject* composite_field, - google::protobuf::Message* parent_message) { + Message* parent_message) { return VisitCompositeField( field_descriptor, composite_field, ReleaseChild(parent_message)); } -int InternalReleaseField(CMessage* self, PyObject* composite_field, - PyObject* name) { - const google::protobuf::FieldDescriptor* descriptor = GetFieldDescriptor(self, name); - if (descriptor != NULL) { - return InternalReleaseFieldByDescriptor( - descriptor, composite_field, self->message); - } - - return 0; -} - PyObject* ClearFieldByDescriptor( CMessage* self, - const google::protobuf::FieldDescriptor* descriptor) { + const FieldDescriptor* descriptor) { if (!CheckFieldBelongsToMessage(descriptor, self->message)) { return NULL; } @@ -1144,25 +1163,23 @@ PyObject* ClearFieldByDescriptor( } PyObject* ClearField(CMessage* self, PyObject* arg) { - char* field_name; if (!PyString_Check(arg)) { PyErr_SetString(PyExc_TypeError, "field name must be a string"); return NULL; } #if PY_MAJOR_VERSION < 3 - if (PyString_AsStringAndSize(arg, &field_name, NULL) < 0) { - return NULL; - } + const char* field_name = PyString_AS_STRING(arg); + Py_ssize_t size = PyString_GET_SIZE(arg); #else - field_name = PyUnicode_AsUTF8(arg); + Py_ssize_t size; + const char* field_name = PyUnicode_AsUTF8AndSize(arg, &size); #endif AssureWritable(self); - google::protobuf::Message* message = self->message; - const google::protobuf::Descriptor* descriptor = message->GetDescriptor(); + Message* message = self->message; ScopedPyObjectPtr arg_in_oneof; bool is_in_oneof; - const google::protobuf::FieldDescriptor* field_descriptor = - FindFieldWithOneofs(message, field_name, &is_in_oneof); + const FieldDescriptor* field_descriptor = + FindFieldWithOneofs(message, string(field_name, size), &is_in_oneof); if (field_descriptor == NULL) { if (!is_in_oneof) { PyErr_Format(PyExc_ValueError, @@ -1172,24 +1189,27 @@ PyObject* ClearField(CMessage* self, PyObject* arg) { Py_RETURN_NONE; } } else if (is_in_oneof) { - arg_in_oneof.reset(PyString_FromString(field_descriptor->name().c_str())); + const string& name = field_descriptor->name(); + arg_in_oneof.reset(PyString_FromStringAndSize(name.c_str(), name.size())); arg = arg_in_oneof.get(); } - PyObject* composite_field = PyDict_GetItem(self->composite_fields, - arg); + PyObject* composite_field = self->composite_fields ? + PyDict_GetItem(self->composite_fields, arg) : NULL; // Only release the field if there's a possibility that there are // references to it. if (composite_field != NULL) { - if (InternalReleaseField(self, composite_field, arg) < 0) { + if (InternalReleaseFieldByDescriptor(field_descriptor, + composite_field, message) < 0) { return NULL; } PyDict_DelItem(self->composite_fields, arg); } message->GetReflection()->ClearField(message, field_descriptor); - if (field_descriptor->cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_ENUM) { - google::protobuf::UnknownFieldSet* unknown_field_set = + if (field_descriptor->cpp_type() == FieldDescriptor::CPPTYPE_ENUM && + !message->GetReflection()->SupportsUnknownEnumValues()) { + UnknownFieldSet* unknown_field_set = message->GetReflection()->MutableUnknownFields(message); unknown_field_set->DeleteByNumber(field_descriptor->number()); } @@ -1212,7 +1232,9 @@ PyObject* Clear(CMessage* self) { } self->extensions = extension_dict; } - PyDict_Clear(self->composite_fields); + if (self->composite_fields) { + PyDict_Clear(self->composite_fields); + } self->message->Clear(); Py_RETURN_NONE; } @@ -1242,7 +1264,27 @@ static PyObject* SerializeToString(CMessage* self, PyObject* args) { if (joined == NULL) { return NULL; } - PyErr_Format(EncodeError_class, "Message %s is missing required fields: %s", + + // TODO(haberman): this is a (hopefully temporary) hack. The unit testing + // infrastructure reloads all pure-Python modules for every test, but not + // C++ modules (because that's generally impossible: + // http://bugs.python.org/issue1144263). But if we cache EncodeError, we'll + // return the EncodeError from a previous load of the module, which won't + // match a user's attempt to catch EncodeError. So we have to look it up + // again every time. + ScopedPyObjectPtr message_module(PyImport_ImportModule( + "google.protobuf.message")); + if (message_module.get() == NULL) { + return NULL; + } + + ScopedPyObjectPtr encode_error( + PyObject_GetAttrString(message_module, "EncodeError")); + if (encode_error.get() == NULL) { + return NULL; + } + PyErr_Format(encode_error.get(), + "Message %s is missing required fields: %s", GetMessageName(self).c_str(), PyString_AsString(joined)); return NULL; } @@ -1268,7 +1310,7 @@ static PyObject* SerializePartialToString(CMessage* self) { // Formats proto fields for ascii dumps using python formatting functions where // appropriate. -class PythonFieldValuePrinter : public google::protobuf::TextFormat::FieldValuePrinter { +class PythonFieldValuePrinter : public TextFormat::FieldValuePrinter { public: PythonFieldValuePrinter() : float_holder_(PyFloat_FromDouble(0)) {} @@ -1301,7 +1343,7 @@ class PythonFieldValuePrinter : public google::protobuf::TextFormat::FieldValueP }; static PyObject* ToStr(CMessage* self) { - google::protobuf::TextFormat::Printer printer; + TextFormat::Printer printer; // Passes ownership printer.SetDefaultFieldValuePrinter(new PythonFieldValuePrinter()); printer.SetHideUnknownFields(true); @@ -1383,9 +1425,9 @@ static PyObject* MergeFromString(CMessage* self, PyObject* arg) { } AssureWritable(self); - google::protobuf::io::CodedInputStream input( + io::CodedInputStream input( reinterpret_cast<const uint8*>(data), data_length); - input.SetExtensionRegistry(descriptor_pool->pool, global_message_factory); + input.SetExtensionRegistry(GetDescriptorPool()->pool, message_factory); bool success = self->message->MergePartialFromCodedStream(&input); if (success) { return PyInt_FromLong(input.CurrentPosition()); @@ -1412,10 +1454,22 @@ static PyObject* RegisterExtension(PyObject* cls, if (message_descriptor == NULL) { return NULL; } - if (PyObject_SetAttrString(extension_handle, "containing_type", - message_descriptor) < 0) { + + const FieldDescriptor* descriptor = + GetExtensionDescriptor(extension_handle); + if (descriptor == NULL) { return NULL; } + const Descriptor* cmessage_descriptor = GetMessageDescriptor( + reinterpret_cast<PyTypeObject*>(cls)); + + if (cmessage_descriptor != descriptor->containing_type()) { + if (PyObject_SetAttrString(extension_handle, "containing_type", + message_descriptor) < 0) { + return NULL; + } + } + ScopedPyObjectPtr extensions_by_name( PyObject_GetAttr(cls, k_extensions_by_name)); if (extensions_by_name == NULL) { @@ -1426,6 +1480,20 @@ static PyObject* RegisterExtension(PyObject* cls, if (full_name == NULL) { return NULL; } + + // If the extension was already registered, check that it is the same. + PyObject* existing_extension = PyDict_GetItem(extensions_by_name, full_name); + if (existing_extension != NULL) { + const FieldDescriptor* existing_extension_descriptor = + GetExtensionDescriptor(existing_extension); + if (existing_extension_descriptor != descriptor) { + PyErr_SetString(PyExc_ValueError, "Double registration of Extensions"); + return NULL; + } + // Nothing else to do. + Py_RETURN_NONE; + } + if (PyDict_SetItem(extensions_by_name, full_name, extension_handle) < 0) { return NULL; } @@ -1445,17 +1513,12 @@ static PyObject* RegisterExtension(PyObject* cls, return NULL; } - const google::protobuf::FieldDescriptor* descriptor = - GetExtensionDescriptor(extension_handle); - if (descriptor == NULL) { - return NULL; - } // Check if it's a message set if (descriptor->is_extension() && descriptor->containing_type()->options().message_set_wire_format() && - descriptor->type() == google::protobuf::FieldDescriptor::TYPE_MESSAGE && + descriptor->type() == FieldDescriptor::TYPE_MESSAGE && descriptor->message_type() == descriptor->extension_scope() && - descriptor->label() == google::protobuf::FieldDescriptor::LABEL_OPTIONAL) { + descriptor->label() == FieldDescriptor::LABEL_OPTIONAL) { ScopedPyObjectPtr message_name(PyString_FromStringAndSize( descriptor->message_type()->full_name().c_str(), descriptor->message_type()->full_name().size())); @@ -1474,53 +1537,36 @@ static PyObject* SetInParent(CMessage* self, PyObject* args) { } static PyObject* WhichOneof(CMessage* self, PyObject* arg) { - char* oneof_name; - if (!PyString_Check(arg)) { - PyErr_SetString(PyExc_TypeError, "field name must be a string"); - return NULL; - } - oneof_name = PyString_AsString(arg); - if (oneof_name == NULL) { + Py_ssize_t name_size; + char *name_data; + if (PyString_AsStringAndSize(arg, &name_data, &name_size) < 0) return NULL; - } - const google::protobuf::OneofDescriptor* oneof_desc = + string oneof_name = string(name_data, name_size); + const OneofDescriptor* oneof_desc = self->message->GetDescriptor()->FindOneofByName(oneof_name); if (oneof_desc == NULL) { PyErr_Format(PyExc_ValueError, - "Protocol message has no oneof \"%s\" field.", oneof_name); + "Protocol message has no oneof \"%s\" field.", + oneof_name.c_str()); return NULL; } - const google::protobuf::FieldDescriptor* field_in_oneof = + const FieldDescriptor* field_in_oneof = self->message->GetReflection()->GetOneofFieldDescriptor( *self->message, oneof_desc); if (field_in_oneof == NULL) { Py_RETURN_NONE; } else { - return PyString_FromString(field_in_oneof->name().c_str()); + const string& name = field_in_oneof->name(); + return PyString_FromStringAndSize(name.c_str(), name.size()); } } static PyObject* ListFields(CMessage* self) { - vector<const google::protobuf::FieldDescriptor*> fields; + vector<const FieldDescriptor*> fields; self->message->GetReflection()->ListFields(*self->message, &fields); - PyObject* descriptor = PyDict_GetItem(Py_TYPE(self)->tp_dict, kDESCRIPTOR); - if (descriptor == NULL) { - return NULL; - } - ScopedPyObjectPtr fields_by_name( - PyObject_GetAttr(descriptor, kfields_by_name)); - if (fields_by_name == NULL) { - return NULL; - } - ScopedPyObjectPtr extensions_by_name(PyObject_GetAttr( - reinterpret_cast<PyObject*>(Py_TYPE(self)), k_extensions_by_name)); - if (extensions_by_name == NULL) { - PyErr_SetString(PyExc_ValueError, "no extensionsbyname"); - return NULL; - } // Normally, the list will be exactly the size of the fields. - PyObject* all_fields = PyList_New(fields.size()); + ScopedPyObjectPtr all_fields(PyList_New(fields.size())); if (all_fields == NULL) { return NULL; } @@ -1532,35 +1578,34 @@ static PyObject* ListFields(CMessage* self) { for (Py_ssize_t i = 0; i < fields.size(); ++i) { ScopedPyObjectPtr t(PyTuple_New(2)); if (t == NULL) { - Py_DECREF(all_fields); return NULL; } if (fields[i]->is_extension()) { - const string& field_name = fields[i]->full_name(); - PyObject* extension_field = PyDict_GetItemString(extensions_by_name, - field_name.c_str()); + ScopedPyObjectPtr extension_field(PyFieldDescriptor_New(fields[i])); if (extension_field == NULL) { - // If we couldn't fetch extension_field, it means the module that - // defines this extension has not been explicitly imported in Python - // code, and the extension hasn't been registered. There's nothing much - // we can do about this, so just skip it in the output to match the - // behavior of the python implementation. + return NULL; + } + // With C++ descriptors, the field can always be retrieved, but for + // unknown extensions which have not been imported in Python code, there + // is no message class and we cannot retrieve the value. + // TODO(amauryfa): consider building the class on the fly! + if (fields[i]->message_type() != NULL && + cdescriptor_pool::GetMessageClass( + GetDescriptorPool(), fields[i]->message_type()) == NULL) { + PyErr_Clear(); continue; } PyObject* extensions = reinterpret_cast<PyObject*>(self->extensions); if (extensions == NULL) { - Py_DECREF(all_fields); return NULL; } // 'extension' reference later stolen by PyTuple_SET_ITEM. PyObject* extension = PyObject_GetItem(extensions, extension_field); if (extension == NULL) { - Py_DECREF(all_fields); return NULL; } - Py_INCREF(extension_field); - PyTuple_SET_ITEM(t.get(), 0, extension_field); + PyTuple_SET_ITEM(t.get(), 0, extension_field.release()); // Steals reference to 'extension' PyTuple_SET_ITEM(t.get(), 1, extension); } else { @@ -1569,35 +1614,30 @@ static PyObject* ListFields(CMessage* self) { field_name.c_str(), field_name.length())); if (py_field_name == NULL) { PyErr_SetString(PyExc_ValueError, "bad string"); - Py_DECREF(all_fields); return NULL; } - PyObject* field_descriptor = - PyDict_GetItem(fields_by_name, py_field_name); + ScopedPyObjectPtr field_descriptor(PyFieldDescriptor_New(fields[i])); if (field_descriptor == NULL) { - Py_DECREF(all_fields); return NULL; } PyObject* field_value = GetAttr(self, py_field_name); if (field_value == NULL) { PyErr_SetObject(PyExc_ValueError, py_field_name); - Py_DECREF(all_fields); return NULL; } - Py_INCREF(field_descriptor); - PyTuple_SET_ITEM(t.get(), 0, field_descriptor); + PyTuple_SET_ITEM(t.get(), 0, field_descriptor.release()); PyTuple_SET_ITEM(t.get(), 1, field_value); } - PyList_SET_ITEM(all_fields, actual_size, t.release()); + PyList_SET_ITEM(all_fields.get(), actual_size, t.release()); ++actual_size; } - Py_SIZE(all_fields) = actual_size; - return all_fields; + Py_SIZE(all_fields.get()) = actual_size; + return all_fields.release(); } PyObject* FindInitializationErrors(CMessage* self) { - google::protobuf::Message* message = self->message; + Message* message = self->message; vector<string> errors; message->FindInitializationErrors(&errors); @@ -1645,9 +1685,9 @@ static PyObject* RichCompare(CMessage* self, PyObject* other, int opid) { PyObject* InternalGetScalar( CMessage* self, - const google::protobuf::FieldDescriptor* field_descriptor) { - google::protobuf::Message* message = self->message; - const google::protobuf::Reflection* reflection = message->GetReflection(); + const FieldDescriptor* field_descriptor) { + Message* message = self->message; + const Reflection* reflection = message->GetReflection(); if (!CheckFieldBelongsToMessage(field_descriptor, message)) { return NULL; @@ -1655,50 +1695,51 @@ PyObject* InternalGetScalar( PyObject* result = NULL; switch (field_descriptor->cpp_type()) { - case google::protobuf::FieldDescriptor::CPPTYPE_INT32: { + case FieldDescriptor::CPPTYPE_INT32: { int32 value = reflection->GetInt32(*message, field_descriptor); result = PyInt_FromLong(value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_INT64: { + case FieldDescriptor::CPPTYPE_INT64: { int64 value = reflection->GetInt64(*message, field_descriptor); result = PyLong_FromLongLong(value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: { + case FieldDescriptor::CPPTYPE_UINT32: { uint32 value = reflection->GetUInt32(*message, field_descriptor); result = PyInt_FromSize_t(value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: { + case FieldDescriptor::CPPTYPE_UINT64: { uint64 value = reflection->GetUInt64(*message, field_descriptor); result = PyLong_FromUnsignedLongLong(value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: { + case FieldDescriptor::CPPTYPE_FLOAT: { float value = reflection->GetFloat(*message, field_descriptor); result = PyFloat_FromDouble(value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: { + case FieldDescriptor::CPPTYPE_DOUBLE: { double value = reflection->GetDouble(*message, field_descriptor); result = PyFloat_FromDouble(value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: { + case FieldDescriptor::CPPTYPE_BOOL: { bool value = reflection->GetBool(*message, field_descriptor); result = PyBool_FromLong(value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_STRING: { + case FieldDescriptor::CPPTYPE_STRING: { string value = reflection->GetString(*message, field_descriptor); result = ToStringObject(field_descriptor, value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: { - if (!message->GetReflection()->HasField(*message, field_descriptor)) { + case FieldDescriptor::CPPTYPE_ENUM: { + if (!message->GetReflection()->SupportsUnknownEnumValues() && + !message->GetReflection()->HasField(*message, field_descriptor)) { // Look for the value in the unknown fields. - google::protobuf::UnknownFieldSet* unknown_field_set = + UnknownFieldSet* unknown_field_set = message->GetReflection()->MutableUnknownFields(message); for (int i = 0; i < unknown_field_set->field_count(); ++i) { if (unknown_field_set->field(i).number() == @@ -1710,7 +1751,7 @@ PyObject* InternalGetScalar( } if (result == NULL) { - const google::protobuf::EnumValueDescriptor* enum_value = + const EnumValueDescriptor* enum_value = message->GetReflection()->GetEnum(*message, field_descriptor); result = PyInt_FromLong(enum_value->number()); } @@ -1726,13 +1767,13 @@ PyObject* InternalGetScalar( } PyObject* InternalGetSubMessage( - CMessage* self, const google::protobuf::FieldDescriptor* field_descriptor) { - const google::protobuf::Reflection* reflection = self->message->GetReflection(); - const google::protobuf::Message& sub_message = reflection->GetMessage( - *self->message, field_descriptor, global_message_factory); + CMessage* self, const FieldDescriptor* field_descriptor) { + const Reflection* reflection = self->message->GetReflection(); + const Message& sub_message = reflection->GetMessage( + *self->message, field_descriptor, message_factory); PyObject *message_class = cdescriptor_pool::GetMessageClass( - descriptor_pool, field_descriptor->message_type()); + GetDescriptorPool(), field_descriptor->message_type()); if (message_class == NULL) { return NULL; } @@ -1747,17 +1788,17 @@ PyObject* InternalGetSubMessage( cmsg->parent = self; cmsg->parent_field_descriptor = field_descriptor; cmsg->read_only = !reflection->HasField(*self->message, field_descriptor); - cmsg->message = const_cast<google::protobuf::Message*>(&sub_message); + cmsg->message = const_cast<Message*>(&sub_message); return reinterpret_cast<PyObject*>(cmsg); } int InternalSetScalar( CMessage* self, - const google::protobuf::FieldDescriptor* field_descriptor, + const FieldDescriptor* field_descriptor, PyObject* arg) { - google::protobuf::Message* message = self->message; - const google::protobuf::Reflection* reflection = message->GetReflection(); + Message* message = self->message; + const Reflection* reflection = message->GetReflection(); if (!CheckFieldBelongsToMessage(field_descriptor, message)) { return -1; @@ -1768,59 +1809,62 @@ int InternalSetScalar( } switch (field_descriptor->cpp_type()) { - case google::protobuf::FieldDescriptor::CPPTYPE_INT32: { + case FieldDescriptor::CPPTYPE_INT32: { GOOGLE_CHECK_GET_INT32(arg, value, -1); reflection->SetInt32(message, field_descriptor, value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_INT64: { + case FieldDescriptor::CPPTYPE_INT64: { GOOGLE_CHECK_GET_INT64(arg, value, -1); reflection->SetInt64(message, field_descriptor, value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: { + case FieldDescriptor::CPPTYPE_UINT32: { GOOGLE_CHECK_GET_UINT32(arg, value, -1); reflection->SetUInt32(message, field_descriptor, value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: { + case FieldDescriptor::CPPTYPE_UINT64: { GOOGLE_CHECK_GET_UINT64(arg, value, -1); reflection->SetUInt64(message, field_descriptor, value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: { + case FieldDescriptor::CPPTYPE_FLOAT: { GOOGLE_CHECK_GET_FLOAT(arg, value, -1); reflection->SetFloat(message, field_descriptor, value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: { + case FieldDescriptor::CPPTYPE_DOUBLE: { GOOGLE_CHECK_GET_DOUBLE(arg, value, -1); reflection->SetDouble(message, field_descriptor, value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: { + case FieldDescriptor::CPPTYPE_BOOL: { GOOGLE_CHECK_GET_BOOL(arg, value, -1); reflection->SetBool(message, field_descriptor, value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_STRING: { + case FieldDescriptor::CPPTYPE_STRING: { if (!CheckAndSetString( arg, message, field_descriptor, reflection, false, -1)) { return -1; } break; } - case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: { + case FieldDescriptor::CPPTYPE_ENUM: { GOOGLE_CHECK_GET_INT32(arg, value, -1); - const google::protobuf::EnumDescriptor* enum_descriptor = - field_descriptor->enum_type(); - const google::protobuf::EnumValueDescriptor* enum_value = - enum_descriptor->FindValueByNumber(value); - if (enum_value != NULL) { - reflection->SetEnum(message, field_descriptor, enum_value); + if (reflection->SupportsUnknownEnumValues()) { + reflection->SetEnumValue(message, field_descriptor, value); } else { - PyErr_Format(PyExc_ValueError, "Unknown enum value: %d", value); - return -1; + const EnumDescriptor* enum_descriptor = field_descriptor->enum_type(); + const EnumValueDescriptor* enum_value = + enum_descriptor->FindValueByNumber(value); + if (enum_value != NULL) { + reflection->SetEnum(message, field_descriptor, enum_value); + } else { + PyErr_Format(PyExc_ValueError, "Unknown enum value: %d", value); + return -1; + } } break; } @@ -1851,14 +1895,35 @@ PyObject* FromString(PyTypeObject* cls, PyObject* serialized) { return py_cmsg; } +// Add the number of a field descriptor to the containing message class. +// Equivalent to: +// _cls.<field>_FIELD_NUMBER = <number> +static bool AddFieldNumberToClass( + PyObject* cls, const FieldDescriptor* field_descriptor) { + string constant_name = field_descriptor->name() + "_FIELD_NUMBER"; + UpperString(&constant_name); + ScopedPyObjectPtr attr_name(PyString_FromStringAndSize( + constant_name.c_str(), constant_name.size())); + if (attr_name == NULL) { + return false; + } + ScopedPyObjectPtr number(PyInt_FromLong(field_descriptor->number())); + if (number == NULL) { + return false; + } + if (PyObject_SetAttr(cls, attr_name, number) == -1) { + return false; + } + return true; +} + // Finalize the creation of the Message class. // Called from its metaclass: GeneratedProtocolMessageType.__init__(). -static PyObject* AddDescriptors(PyTypeObject* cls, - PyObject* descriptor) { - const google::protobuf::Descriptor* message_descriptor = +static PyObject* AddDescriptors(PyObject* cls, PyObject* descriptor) { + const Descriptor* message_descriptor = cdescriptor_pool::RegisterMessageClass( - descriptor_pool, reinterpret_cast<PyObject*>(cls), descriptor); + GetDescriptorPool(), cls, descriptor); if (message_descriptor == NULL) { return NULL; } @@ -1867,169 +1932,80 @@ static PyObject* AddDescriptors(PyTypeObject* cls, // classes will register themselves in this class. if (message_descriptor->extension_range_count() > 0) { ScopedPyObjectPtr by_name(PyDict_New()); - if (PyObject_SetAttr(reinterpret_cast<PyObject*>(cls), - k_extensions_by_name, by_name) < 0) { + if (PyObject_SetAttr(cls, k_extensions_by_name, by_name) < 0) { return NULL; } ScopedPyObjectPtr by_number(PyDict_New()); - if (PyObject_SetAttr(reinterpret_cast<PyObject*>(cls), - k_extensions_by_number, by_number) < 0) { + if (PyObject_SetAttr(cls, k_extensions_by_number, by_number) < 0) { return NULL; } } - ScopedPyObjectPtr fields(PyObject_GetAttrString(descriptor, "fields")); - if (fields == NULL) { - return NULL; - } - - ScopedPyObjectPtr _NUMBER_string(PyString_FromString("_FIELD_NUMBER")); - if (_NUMBER_string == NULL) { - return NULL; - } - - const Py_ssize_t fields_size = PyList_GET_SIZE(fields.get()); - for (int i = 0; i < fields_size; ++i) { - PyObject* field = PyList_GET_ITEM(fields.get(), i); - ScopedPyObjectPtr field_name(PyObject_GetAttr(field, kname)); - ScopedPyObjectPtr full_field_name(PyObject_GetAttr(field, kfull_name)); - if (field_name == NULL || full_field_name == NULL) { - PyErr_SetString(PyExc_TypeError, "Name is null"); - return NULL; - } - - ScopedPyObjectPtr field_descriptor( - cdescriptor_pool::FindFieldByName(descriptor_pool, full_field_name)); - if (field_descriptor == NULL) { - PyErr_SetString(PyExc_TypeError, "Couldn't find field"); - return NULL; - } - CFieldDescriptor* cfield_descriptor = reinterpret_cast<CFieldDescriptor*>( - field_descriptor.get()); - - // The FieldDescriptor's name field might either be of type bytes or - // of type unicode, depending on whether the FieldDescriptor was - // parsed from a serialized message or read from the - // <message>_pb2.py module. - ScopedPyObjectPtr field_name_upcased( - PyObject_CallMethod(field_name, "upper", NULL)); - if (field_name_upcased == NULL) { - return NULL; - } - - ScopedPyObjectPtr field_number_name(PyObject_CallMethod( - field_name_upcased, "__add__", "(O)", _NUMBER_string.get())); - if (field_number_name == NULL) { - return NULL; - } - - ScopedPyObjectPtr number(PyInt_FromLong( - cfield_descriptor->descriptor->number())); - if (number == NULL) { - return NULL; - } - if (PyObject_SetAttr(reinterpret_cast<PyObject*>(cls), - field_number_name, number) == -1) { + // For each field set: cls.<field>_FIELD_NUMBER = <number> + for (int i = 0; i < message_descriptor->field_count(); ++i) { + if (!AddFieldNumberToClass(cls, message_descriptor->field(i))) { return NULL; } } - // Enum Values - ScopedPyObjectPtr enum_types(PyObject_GetAttrString(descriptor, - "enum_types")); - if (enum_types == NULL) { - return NULL; - } - ScopedPyObjectPtr type_iter(PyObject_GetIter(enum_types)); - if (type_iter == NULL) { - return NULL; - } - ScopedPyObjectPtr enum_type; - while ((enum_type.reset(PyIter_Next(type_iter))) != NULL) { + // For each enum set cls.<enum name> = EnumTypeWrapper(<enum descriptor>). + // + // The enum descriptor we get from + // <messagedescriptor>.enum_types_by_name[name] + // which was built previously. + for (int i = 0; i < message_descriptor->enum_type_count(); ++i) { + const EnumDescriptor* enum_descriptor = message_descriptor->enum_type(i); + ScopedPyObjectPtr enum_type(PyEnumDescriptor_New(enum_descriptor)); + if (enum_type == NULL) { + return NULL; + } + // Add wrapped enum type to message class. ScopedPyObjectPtr wrapped(PyObject_CallFunctionObjArgs( EnumTypeWrapper_class, enum_type.get(), NULL)); if (wrapped == NULL) { return NULL; } - ScopedPyObjectPtr enum_name(PyObject_GetAttr(enum_type, kname)); - if (enum_name == NULL) { - return NULL; - } - if (PyObject_SetAttr(reinterpret_cast<PyObject*>(cls), - enum_name, wrapped) == -1) { + if (PyObject_SetAttrString( + cls, enum_descriptor->name().c_str(), wrapped) == -1) { return NULL; } - ScopedPyObjectPtr enum_values(PyObject_GetAttrString(enum_type, "values")); - if (enum_values == NULL) { - return NULL; - } - ScopedPyObjectPtr values_iter(PyObject_GetIter(enum_values)); - if (values_iter == NULL) { - return NULL; - } - ScopedPyObjectPtr enum_value; - while ((enum_value.reset(PyIter_Next(values_iter))) != NULL) { - ScopedPyObjectPtr value_name(PyObject_GetAttr(enum_value, kname)); - if (value_name == NULL) { - return NULL; - } - ScopedPyObjectPtr value_number(PyObject_GetAttrString(enum_value, - "number")); + // For each enum value add cls.<name> = <number> + for (int j = 0; j < enum_descriptor->value_count(); ++j) { + const EnumValueDescriptor* enum_value_descriptor = + enum_descriptor->value(j); + ScopedPyObjectPtr value_number(PyInt_FromLong( + enum_value_descriptor->number())); if (value_number == NULL) { return NULL; } - if (PyObject_SetAttr(reinterpret_cast<PyObject*>(cls), - value_name, value_number) == -1) { + if (PyObject_SetAttrString( + cls, enum_value_descriptor->name().c_str(), value_number) == -1) { return NULL; } } - if (PyErr_Occurred()) { // If PyIter_Next failed - return NULL; - } - } - if (PyErr_Occurred()) { // If PyIter_Next failed - return NULL; } - ScopedPyObjectPtr extension_dict( - PyObject_GetAttr(descriptor, kextensions_by_name)); - if (extension_dict == NULL || !PyDict_Check(extension_dict)) { - PyErr_SetString(PyExc_TypeError, "extensions_by_name not a dict"); - return NULL; - } - Py_ssize_t pos = 0; - PyObject* extension_name; - PyObject* extension_field; - - while (PyDict_Next(extension_dict, &pos, &extension_name, &extension_field)) { - if (PyObject_SetAttr(reinterpret_cast<PyObject*>(cls), - extension_name, extension_field) == -1) { - return NULL; - } - const google::protobuf::FieldDescriptor* field_descriptor = - GetExtensionDescriptor(extension_field); - if (field_descriptor == NULL) { + // For each extension set cls.<extension name> = <extension descriptor>. + // + // Extension descriptors come from + // <message descriptor>.extensions_by_name[name] + // which was defined previously. + for (int i = 0; i < message_descriptor->extension_count(); ++i) { + const google::protobuf::FieldDescriptor* field = message_descriptor->extension(i); + ScopedPyObjectPtr extension_field(PyFieldDescriptor_New(field)); + if (extension_field == NULL) { return NULL; } - ScopedPyObjectPtr field_name_upcased( - PyObject_CallMethod(extension_name, "upper", NULL)); - if (field_name_upcased == NULL) { + // Add the extension field to the message class. + if (PyObject_SetAttrString( + cls, field->name().c_str(), extension_field) == -1) { return NULL; } - ScopedPyObjectPtr field_number_name(PyObject_CallMethod( - field_name_upcased, "__add__", "(O)", _NUMBER_string.get())); - if (field_number_name == NULL) { - return NULL; - } - ScopedPyObjectPtr number(PyInt_FromLong( - field_descriptor->number())); - if (number == NULL) { - return NULL; - } - if (PyObject_SetAttr(reinterpret_cast<PyObject*>(cls), - field_number_name, number) == -1) { + + // For each extension set cls.<extension name>_FIELD_NUMBER = <number>. + if (!AddFieldNumberToClass(cls, field)) { return NULL; } } @@ -2121,12 +2097,35 @@ PyObject* SetState(CMessage* self, PyObject* state) { } // CMessage static methods: +PyObject* _GetMessageDescriptor(PyObject* unused, PyObject* arg) { + return cdescriptor_pool::FindMessageByName(GetDescriptorPool(), arg); +} + PyObject* _GetFieldDescriptor(PyObject* unused, PyObject* arg) { - return cdescriptor_pool::FindFieldByName(descriptor_pool, arg); + return cdescriptor_pool::FindFieldByName(GetDescriptorPool(), arg); } PyObject* _GetExtensionDescriptor(PyObject* unused, PyObject* arg) { - return cdescriptor_pool::FindExtensionByName(descriptor_pool, arg); + return cdescriptor_pool::FindExtensionByName(GetDescriptorPool(), arg); +} + +PyObject* _GetEnumDescriptor(PyObject* unused, PyObject* arg) { + return cdescriptor_pool::FindEnumTypeByName(GetDescriptorPool(), arg); +} + +PyObject* _GetOneofDescriptor(PyObject* unused, PyObject* arg) { + return cdescriptor_pool::FindOneofByName(GetDescriptorPool(), arg); +} + +PyObject* _CheckCalledFromGeneratedFile(PyObject* unused, + PyObject* unused_arg) { + if (!_CalledFromGeneratedFile(1)) { + PyErr_SetString(PyExc_TypeError, + "Descriptors should not be created directly, " + "but only retrieved from their parent."); + return NULL; + } + Py_RETURN_NONE; } static PyMemberDef Members[] = { @@ -2191,91 +2190,102 @@ static PyMethodDef Methods[] = { // Static Methods. { "_BuildFile", (PyCFunction)Python_BuildFile, METH_O | METH_STATIC, "Registers a new protocol buffer file in the global C++ descriptor pool." }, + { "_GetMessageDescriptor", (PyCFunction)_GetMessageDescriptor, + METH_O | METH_STATIC, "Finds a message descriptor in the message pool." }, { "_GetFieldDescriptor", (PyCFunction)_GetFieldDescriptor, METH_O | METH_STATIC, "Finds a field descriptor in the message pool." }, { "_GetExtensionDescriptor", (PyCFunction)_GetExtensionDescriptor, METH_O | METH_STATIC, "Finds a extension descriptor in the message pool." }, + { "_GetEnumDescriptor", (PyCFunction)_GetEnumDescriptor, + METH_O | METH_STATIC, + "Finds an enum descriptor in the message pool." }, + { "_GetOneofDescriptor", (PyCFunction)_GetOneofDescriptor, + METH_O | METH_STATIC, + "Finds an oneof descriptor in the message pool." }, + { "_CheckCalledFromGeneratedFile", (PyCFunction)_CheckCalledFromGeneratedFile, + METH_NOARGS | METH_STATIC, + "Raises TypeError if the caller is not in a _pb2.py file."}, { NULL, NULL} }; +static bool SetCompositeField( + CMessage* self, PyObject* name, PyObject* value) { + if (self->composite_fields == NULL) { + self->composite_fields = PyDict_New(); + if (self->composite_fields == NULL) { + return false; + } + } + return PyDict_SetItem(self->composite_fields, name, value) == 0; +} + PyObject* GetAttr(CMessage* self, PyObject* name) { - PyObject* value = PyDict_GetItem(self->composite_fields, name); + PyObject* value = self->composite_fields ? + PyDict_GetItem(self->composite_fields, name) : NULL; if (value != NULL) { Py_INCREF(value); return value; } - const google::protobuf::FieldDescriptor* field_descriptor = GetFieldDescriptor( - self, name); - if (field_descriptor != NULL) { - if (field_descriptor->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) { - if (field_descriptor->cpp_type() == - google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { - PyObject *message_class = cdescriptor_pool::GetMessageClass( - descriptor_pool, field_descriptor->message_type()); - if (message_class == NULL) { - return NULL; - } - PyObject* py_container = repeated_composite_container::NewContainer( - self, field_descriptor, message_class); - if (py_container == NULL) { - return NULL; - } - if (PyDict_SetItem(self->composite_fields, name, py_container) < 0) { - Py_DECREF(py_container); - return NULL; - } - return py_container; - } else { - PyObject* py_container = repeated_scalar_container::NewContainer( - self, field_descriptor); - if (py_container == NULL) { - return NULL; - } - if (PyDict_SetItem(self->composite_fields, name, py_container) < 0) { - Py_DECREF(py_container); - return NULL; - } - return py_container; + const FieldDescriptor* field_descriptor = GetFieldDescriptor(self, name); + if (field_descriptor == NULL) { + return CMessage_Type.tp_base->tp_getattro( + reinterpret_cast<PyObject*>(self), name); + } + + if (field_descriptor->label() == FieldDescriptor::LABEL_REPEATED) { + PyObject* py_container = NULL; + if (field_descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + PyObject *message_class = cdescriptor_pool::GetMessageClass( + GetDescriptorPool(), field_descriptor->message_type()); + if (message_class == NULL) { + return NULL; } + py_container = repeated_composite_container::NewContainer( + self, field_descriptor, message_class); } else { - if (field_descriptor->cpp_type() == - google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { - PyObject* sub_message = InternalGetSubMessage(self, field_descriptor); - if (PyDict_SetItem(self->composite_fields, name, sub_message) < 0) { - Py_DECREF(sub_message); - return NULL; - } - return sub_message; - } else { - return InternalGetScalar(self, field_descriptor); - } + py_container = repeated_scalar_container::NewContainer( + self, field_descriptor); + } + if (py_container == NULL) { + return NULL; } + if (!SetCompositeField(self, name, py_container)) { + Py_DECREF(py_container); + return NULL; + } + return py_container; } - return CMessage_Type.tp_base->tp_getattro(reinterpret_cast<PyObject*>(self), - name); + if (field_descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + PyObject* sub_message = InternalGetSubMessage(self, field_descriptor); + if (!SetCompositeField(self, name, sub_message)) { + Py_DECREF(sub_message); + return NULL; + } + return sub_message; + } + + return InternalGetScalar(self, field_descriptor); } int SetAttr(CMessage* self, PyObject* name, PyObject* value) { - if (PyDict_Contains(self->composite_fields, name)) { + if (self->composite_fields && PyDict_Contains(self->composite_fields, name)) { PyErr_SetString(PyExc_TypeError, "Can't set composite field"); return -1; } - const google::protobuf::FieldDescriptor* field_descriptor = - GetFieldDescriptor(self, name); + const FieldDescriptor* field_descriptor = GetFieldDescriptor(self, name); if (field_descriptor != NULL) { AssureWritable(self); - if (field_descriptor->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) { + if (field_descriptor->label() == FieldDescriptor::LABEL_REPEATED) { PyErr_Format(PyExc_AttributeError, "Assignment not allowed to repeated " "field \"%s\" in protocol message object.", field_descriptor->name().c_str()); return -1; } else { - if (field_descriptor->cpp_type() == - google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + if (field_descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { PyErr_Format(PyExc_AttributeError, "Assignment not allowed to " "field \"%s\" in protocol message object.", field_descriptor->name().c_str()); @@ -2294,8 +2304,8 @@ int SetAttr(CMessage* self, PyObject* name, PyObject* value) { PyTypeObject CMessage_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - "google.protobuf.internal." - "cpp._message.CMessage", // tp_name + "google.protobuf." + "pyext._message.CMessage", // tp_name sizeof(CMessage), // tp_basicsize 0, // tp_itemsize (destructor)cmessage::Dealloc, // tp_dealloc @@ -2339,7 +2349,7 @@ PyTypeObject CMessage_Type = { const Message* (*GetCProtoInsidePyProtoPtr)(PyObject* msg); Message* (*MutableCProtoInsidePyProtoPtr)(PyObject* msg); -static const google::protobuf::Message* GetCProtoInsidePyProtoImpl(PyObject* msg) { +static const Message* GetCProtoInsidePyProtoImpl(PyObject* msg) { if (!PyObject_TypeCheck(msg, &CMessage_Type)) { return NULL; } @@ -2347,12 +2357,12 @@ static const google::protobuf::Message* GetCProtoInsidePyProtoImpl(PyObject* msg return cmsg->message; } -static google::protobuf::Message* MutableCProtoInsidePyProtoImpl(PyObject* msg) { +static Message* MutableCProtoInsidePyProtoImpl(PyObject* msg) { if (!PyObject_TypeCheck(msg, &CMessage_Type)) { return NULL; } CMessage* cmsg = reinterpret_cast<CMessage*>(msg); - if (PyDict_Size(cmsg->composite_fields) != 0 || + if ((cmsg->composite_fields && PyDict_Size(cmsg->composite_fields) != 0) || (cmsg->extensions != NULL && PyDict_Size(cmsg->extensions->values) != 0)) { // There is currently no way of accurately syncing arbitrary changes to @@ -2387,29 +2397,35 @@ void InitGlobals() { kDESCRIPTOR = PyString_FromString("DESCRIPTOR"); k_cdescriptor = PyString_FromString("_cdescriptor"); kfull_name = PyString_FromString("full_name"); - kextensions_by_name = PyString_FromString("extensions_by_name"); k_extensions_by_name = PyString_FromString("_extensions_by_name"); k_extensions_by_number = PyString_FromString("_extensions_by_number"); - kname = PyString_FromString("name"); - kfields_by_name = PyString_FromString("fields_by_name"); - descriptor_pool = cdescriptor_pool::NewDescriptorPool(); - - global_message_factory = new DynamicMessageFactory(descriptor_pool->pool); - global_message_factory->SetDelegateToGeneratedFactory(true); + message_factory = new DynamicMessageFactory(GetDescriptorPool()->pool); + message_factory->SetDelegateToGeneratedFactory(true); } bool InitProto2MessageModule(PyObject *m) { + // Initialize types and globals in descriptor.cc + if (!InitDescriptor()) { + return false; + } + + // Initialize types and globals in descriptor_pool.cc + if (!InitDescriptorPool()) { + return false; + } + + // Initialize constants defined in this file. InitGlobals(); - google::protobuf::python::CMessage_Type.tp_hash = PyObject_HashNotImplemented; - if (PyType_Ready(&google::protobuf::python::CMessage_Type) < 0) { + CMessage_Type.tp_hash = PyObject_HashNotImplemented; + if (PyType_Ready(&CMessage_Type) < 0) { return false; } // DESCRIPTOR is set on each protocol buffer message class elsewhere, but set // it here as well to document that subclasses need to set it. - PyDict_SetItem(google::protobuf::python::CMessage_Type.tp_dict, kDESCRIPTOR, Py_None); + PyDict_SetItem(CMessage_Type.tp_dict, kDESCRIPTOR, Py_None); // Subclasses with message extensions will override _extensions_by_name and // _extensions_by_number with fresh mutable dictionaries in AddDescriptors. // All other classes can share this same immutable mapping. @@ -2421,58 +2437,69 @@ bool InitProto2MessageModule(PyObject *m) { if (immutable_dict == NULL) { return false; } - if (PyDict_SetItem(google::protobuf::python::CMessage_Type.tp_dict, + if (PyDict_SetItem(CMessage_Type.tp_dict, k_extensions_by_name, immutable_dict) < 0) { return false; } - if (PyDict_SetItem(google::protobuf::python::CMessage_Type.tp_dict, + if (PyDict_SetItem(CMessage_Type.tp_dict, k_extensions_by_number, immutable_dict) < 0) { return false; } - PyModule_AddObject(m, "Message", reinterpret_cast<PyObject*>( - &google::protobuf::python::CMessage_Type)); + PyModule_AddObject(m, "Message", reinterpret_cast<PyObject*>(&CMessage_Type)); - google::protobuf::python::RepeatedScalarContainer_Type.tp_hash = + RepeatedScalarContainer_Type.tp_hash = PyObject_HashNotImplemented; - if (PyType_Ready(&google::protobuf::python::RepeatedScalarContainer_Type) < 0) { + if (PyType_Ready(&RepeatedScalarContainer_Type) < 0) { return false; } PyModule_AddObject(m, "RepeatedScalarContainer", reinterpret_cast<PyObject*>( - &google::protobuf::python::RepeatedScalarContainer_Type)); + &RepeatedScalarContainer_Type)); - google::protobuf::python::RepeatedCompositeContainer_Type.tp_hash = - PyObject_HashNotImplemented; - if (PyType_Ready(&google::protobuf::python::RepeatedCompositeContainer_Type) < 0) { + RepeatedCompositeContainer_Type.tp_hash = PyObject_HashNotImplemented; + if (PyType_Ready(&RepeatedCompositeContainer_Type) < 0) { return false; } PyModule_AddObject( m, "RepeatedCompositeContainer", reinterpret_cast<PyObject*>( - &google::protobuf::python::RepeatedCompositeContainer_Type)); + &RepeatedCompositeContainer_Type)); - google::protobuf::python::ExtensionDict_Type.tp_hash = PyObject_HashNotImplemented; - if (PyType_Ready(&google::protobuf::python::ExtensionDict_Type) < 0) { + ExtensionDict_Type.tp_hash = PyObject_HashNotImplemented; + if (PyType_Ready(&ExtensionDict_Type) < 0) { return false; } PyModule_AddObject( m, "ExtensionDict", - reinterpret_cast<PyObject*>(&google::protobuf::python::ExtensionDict_Type)); - - if (!google::protobuf::python::InitDescriptor()) { - return false; - } + reinterpret_cast<PyObject*>(&ExtensionDict_Type)); + + // This implementation provides full Descriptor types, we advertise it so that + // descriptor.py can use them in replacement of the Python classes. + PyModule_AddIntConstant(m, "_USE_C_DESCRIPTORS", 1); + + PyModule_AddObject(m, "Descriptor", reinterpret_cast<PyObject*>( + &PyMessageDescriptor_Type)); + PyModule_AddObject(m, "FieldDescriptor", reinterpret_cast<PyObject*>( + &PyFieldDescriptor_Type)); + PyModule_AddObject(m, "EnumDescriptor", reinterpret_cast<PyObject*>( + &PyEnumDescriptor_Type)); + PyModule_AddObject(m, "EnumValueDescriptor", reinterpret_cast<PyObject*>( + &PyEnumValueDescriptor_Type)); + PyModule_AddObject(m, "FileDescriptor", reinterpret_cast<PyObject*>( + &PyFileDescriptor_Type)); + PyModule_AddObject(m, "OneofDescriptor", reinterpret_cast<PyObject*>( + &PyOneofDescriptor_Type)); PyObject* enum_type_wrapper = PyImport_ImportModule( "google.protobuf.internal.enum_type_wrapper"); if (enum_type_wrapper == NULL) { return false; } - google::protobuf::python::EnumTypeWrapper_class = + EnumTypeWrapper_class = PyObject_GetAttrString(enum_type_wrapper, "EnumTypeWrapper"); Py_DECREF(enum_type_wrapper); @@ -2481,25 +2508,20 @@ bool InitProto2MessageModule(PyObject *m) { if (message_module == NULL) { return false; } - google::protobuf::python::EncodeError_class = PyObject_GetAttrString(message_module, - "EncodeError"); - google::protobuf::python::DecodeError_class = PyObject_GetAttrString(message_module, - "DecodeError"); + EncodeError_class = PyObject_GetAttrString(message_module, "EncodeError"); + DecodeError_class = PyObject_GetAttrString(message_module, "DecodeError"); Py_DECREF(message_module); PyObject* pickle_module = PyImport_ImportModule("pickle"); if (pickle_module == NULL) { return false; } - google::protobuf::python::PickleError_class = PyObject_GetAttrString(pickle_module, - "PickleError"); + PickleError_class = PyObject_GetAttrString(pickle_module, "PickleError"); Py_DECREF(pickle_module); // Override {Get,Mutable}CProtoInsidePyProto. - google::protobuf::python::GetCProtoInsidePyProtoPtr = - google::protobuf::python::GetCProtoInsidePyProtoImpl; - google::protobuf::python::MutableCProtoInsidePyProtoPtr = - google::protobuf::python::MutableCProtoInsidePyProtoImpl; + GetCProtoInsidePyProtoPtr = GetCProtoInsidePyProtoImpl; + MutableCProtoInsidePyProtoPtr = MutableCProtoInsidePyProtoImpl; return true; } diff --git a/python/google/protobuf/pyext/message.h b/python/google/protobuf/pyext/message.h index 0fef92a0..2f2da795 100644 --- a/python/google/protobuf/pyext/message.h +++ b/python/google/protobuf/pyext/message.h @@ -42,7 +42,6 @@ #endif #include <string> - namespace google { namespace protobuf { @@ -50,12 +49,12 @@ class Message; class Reflection; class FieldDescriptor; class Descriptor; +class DynamicMessageFactory; using internal::shared_ptr; namespace python { -struct PyDescriptorPool; struct ExtensionDict; typedef struct CMessage { @@ -84,7 +83,7 @@ typedef struct CMessage { // Used together with the parent's message when making a default message // instance mutable. // The pointer is owned by the global DescriptorPool. - const google::protobuf::FieldDescriptor* parent_field_descriptor; + const FieldDescriptor* parent_field_descriptor; // Pointer to the C++ Message object for this CMessage. The // CMessage does not own this pointer. @@ -115,27 +114,26 @@ namespace cmessage { // Internal function to create a new empty Message Python object, but with empty // pointers to the C++ objects. // The caller must fill self->message, self->owner and eventually self->parent. -CMessage* NewEmptyMessage(PyObject* type, - const google::protobuf::Descriptor* descriptor); +CMessage* NewEmptyMessage(PyObject* type, const Descriptor* descriptor); // Release a submessage from its proto tree, making it a new top-level messgae. // A new message will be created if this is a read-only default instance. // // Corresponds to reflection api method ReleaseMessage. -int ReleaseSubMessage(google::protobuf::Message* message, - const google::protobuf::FieldDescriptor* field_descriptor, +int ReleaseSubMessage(Message* message, + const FieldDescriptor* field_descriptor, CMessage* child_cmessage); // Retrieves the C++ descriptor of a Python Extension descriptor. // On error, return NULL with an exception set. -const google::protobuf::FieldDescriptor* GetExtensionDescriptor(PyObject* extension); +const FieldDescriptor* GetExtensionDescriptor(PyObject* extension); // Initializes a new CMessage instance for a submessage. Only called once per // submessage as the result is cached in composite_fields. // // Corresponds to reflection api method GetMessage. PyObject* InternalGetSubMessage( - CMessage* self, const google::protobuf::FieldDescriptor* field_descriptor); + CMessage* self, const FieldDescriptor* field_descriptor); // Deletes a range of C++ submessages in a repeated field (following a // removal in a RepeatedCompositeContainer). @@ -146,20 +144,20 @@ PyObject* InternalGetSubMessage( // by slice will be removed from cmessage_list by this function. // // Corresponds to reflection api method RemoveLast. -int InternalDeleteRepeatedField(google::protobuf::Message* message, - const google::protobuf::FieldDescriptor* field_descriptor, +int InternalDeleteRepeatedField(Message* message, + const FieldDescriptor* field_descriptor, PyObject* slice, PyObject* cmessage_list); // Sets the specified scalar value to the message. int InternalSetScalar(CMessage* self, - const google::protobuf::FieldDescriptor* field_descriptor, + const FieldDescriptor* field_descriptor, PyObject* value); // Retrieves the specified scalar value from the message. // // Returns a new python reference. PyObject* InternalGetScalar(CMessage* self, - const google::protobuf::FieldDescriptor* field_descriptor); + const FieldDescriptor* field_descriptor); // Clears the message, removing all contained data. Extension dictionary and // submessages are released first if there are remaining external references. @@ -175,8 +173,7 @@ PyObject* Clear(CMessage* self); // // Corresponds to reflection api method ClearField. PyObject* ClearFieldByDescriptor( - CMessage* self, - const google::protobuf::FieldDescriptor* descriptor); + CMessage* self, const FieldDescriptor* descriptor); // Clears the data for the given field name. The message is released if there // are any external references. @@ -189,7 +186,7 @@ PyObject* ClearField(CMessage* self, PyObject* arg); // // Corresponds to reflection api method HasField PyObject* HasFieldByDescriptor( - CMessage* self, const google::protobuf::FieldDescriptor* field_descriptor); + CMessage* self, const FieldDescriptor* field_descriptor); // Checks if the message has the named field. // @@ -220,18 +217,16 @@ int SetOwner(CMessage* self, const shared_ptr<Message>& new_owner); int AssureWritable(CMessage* self); -} // namespace cmessage - +DynamicMessageFactory* GetMessageFactory(); -// Retrieve the global descriptor pool owned by the _message module. -PyDescriptorPool* GetDescriptorPool(); +} // namespace cmessage /* Is 64bit */ #define IS_64BIT (SIZEOF_LONG == 8) #define FIELD_IS_REPEATED(field_descriptor) \ - ((field_descriptor)->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) + ((field_descriptor)->label() == FieldDescriptor::LABEL_REPEATED) #define GOOGLE_CHECK_GET_INT32(arg, value, err) \ int32 value; \ @@ -294,18 +289,17 @@ bool CheckAndGetDouble(PyObject* arg, double* value); bool CheckAndGetFloat(PyObject* arg, float* value); bool CheckAndGetBool(PyObject* arg, bool* value); bool CheckAndSetString( - PyObject* arg, google::protobuf::Message* message, - const google::protobuf::FieldDescriptor* descriptor, - const google::protobuf::Reflection* reflection, + PyObject* arg, Message* message, + const FieldDescriptor* descriptor, + const Reflection* reflection, bool append, int index); -PyObject* ToStringObject( - const google::protobuf::FieldDescriptor* descriptor, string value); +PyObject* ToStringObject(const FieldDescriptor* descriptor, string value); // Check if the passed field descriptor belongs to the given message. // If not, return false and set a Python exception (a KeyError) -bool CheckFieldBelongsToMessage(const google::protobuf::FieldDescriptor* field_descriptor, - const google::protobuf::Message* message); +bool CheckFieldBelongsToMessage(const FieldDescriptor* field_descriptor, + const Message* message); extern PyObject* PickleError_class; diff --git a/python/google/protobuf/pyext/message_factory_cpp2_test.py b/python/google/protobuf/pyext/message_factory_cpp2_test.py deleted file mode 100644 index 32ab4f85..00000000 --- a/python/google/protobuf/pyext/message_factory_cpp2_test.py +++ /dev/null @@ -1,56 +0,0 @@ -#! /usr/bin/python -# -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Tests for google.protobuf.message_factory.""" - -import os -os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp' -os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION'] = '2' - -# We must set the implementation version above before the google3 imports. -# pylint: disable=g-import-not-at-top -from google.apputils import basetest -from google.protobuf.internal import api_implementation -# Run all tests from the original module by putting them in our namespace. -# pylint: disable=wildcard-import -from google.protobuf.internal.message_factory_test import * - - -class ConfirmCppApi2Test(basetest.TestCase): - - def testImplementationSetting(self): - self.assertEqual('cpp', api_implementation.Type()) - self.assertEqual(2, api_implementation.Version()) - - -if __name__ == '__main__': - basetest.main() diff --git a/python/google/protobuf/pyext/reflection_cpp2_generated_test.py b/python/google/protobuf/pyext/reflection_cpp2_generated_test.py deleted file mode 100755 index 552efd48..00000000 --- a/python/google/protobuf/pyext/reflection_cpp2_generated_test.py +++ /dev/null @@ -1,94 +0,0 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- -# -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Unittest for reflection.py, which tests the generated C++ implementation.""" - -__author__ = 'jasonh@google.com (Jason Hsueh)' - -import os -os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp' -os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION'] = '2' - -from google.apputils import basetest -from google.protobuf.internal import api_implementation -from google.protobuf.internal import more_extensions_dynamic_pb2 -from google.protobuf.internal import more_extensions_pb2 -from google.protobuf.internal.reflection_test import * - - -class ReflectionCppTest(basetest.TestCase): - def testImplementationSetting(self): - self.assertEqual('cpp', api_implementation.Type()) - self.assertEqual(2, api_implementation.Version()) - - def testExtensionOfGeneratedTypeInDynamicFile(self): - """Tests that a file built dynamically can extend a generated C++ type. - - The C++ implementation uses a DescriptorPool that has the generated - DescriptorPool as an underlay. Typically, a type can only find - extensions in its own pool. With the python C-extension, the generated C++ - extendee may be available, but not the extension. This tests that the - C-extension implements the correct special handling to make such extensions - available. - """ - pb1 = more_extensions_pb2.ExtendedMessage() - # Test that basic accessors work. - self.assertFalse( - pb1.HasExtension(more_extensions_dynamic_pb2.dynamic_int32_extension)) - self.assertFalse( - pb1.HasExtension(more_extensions_dynamic_pb2.dynamic_message_extension)) - pb1.Extensions[more_extensions_dynamic_pb2.dynamic_int32_extension] = 17 - pb1.Extensions[more_extensions_dynamic_pb2.dynamic_message_extension].a = 24 - self.assertTrue( - pb1.HasExtension(more_extensions_dynamic_pb2.dynamic_int32_extension)) - self.assertTrue( - pb1.HasExtension(more_extensions_dynamic_pb2.dynamic_message_extension)) - - # Now serialize the data and parse to a new message. - pb2 = more_extensions_pb2.ExtendedMessage() - pb2.MergeFromString(pb1.SerializeToString()) - - self.assertTrue( - pb2.HasExtension(more_extensions_dynamic_pb2.dynamic_int32_extension)) - self.assertTrue( - pb2.HasExtension(more_extensions_dynamic_pb2.dynamic_message_extension)) - self.assertEqual( - 17, pb2.Extensions[more_extensions_dynamic_pb2.dynamic_int32_extension]) - self.assertEqual( - 24, - pb2.Extensions[more_extensions_dynamic_pb2.dynamic_message_extension].a) - - - -if __name__ == '__main__': - basetest.main() diff --git a/python/google/protobuf/pyext/repeated_composite_container.cc b/python/google/protobuf/pyext/repeated_composite_container.cc index 36fe86ae..abc859d7 100644 --- a/python/google/protobuf/pyext/repeated_composite_container.cc +++ b/python/google/protobuf/pyext/repeated_composite_container.cc @@ -56,8 +56,6 @@ namespace google { namespace protobuf { namespace python { -extern google::protobuf::DynamicMessageFactory* global_message_factory; - namespace repeated_composite_container { // TODO(tibell): We might also want to check: @@ -120,9 +118,9 @@ static int InternalQuickSort(RepeatedCompositeContainer* self, GOOGLE_CHECK_ATTACHED(self); - google::protobuf::Message* message = self->message; - const google::protobuf::Reflection* reflection = message->GetReflection(); - const google::protobuf::FieldDescriptor* descriptor = self->parent_field_descriptor; + Message* message = self->message; + const Reflection* reflection = message->GetReflection(); + const FieldDescriptor* descriptor = self->parent_field_descriptor; Py_ssize_t left; Py_ssize_t right; @@ -199,7 +197,7 @@ static int InternalQuickSort(RepeatedCompositeContainer* self, // len() static Py_ssize_t Length(RepeatedCompositeContainer* self) { - google::protobuf::Message* message = self->message; + Message* message = self->message; if (message != NULL) { return message->GetReflection()->FieldSize(*message, self->parent_field_descriptor); @@ -221,8 +219,8 @@ static int UpdateChildMessages(RepeatedCompositeContainer* self) { // be removed in such a way so there's no need to worry about that. Py_ssize_t message_length = Length(self); Py_ssize_t child_length = PyList_GET_SIZE(self->child_messages); - google::protobuf::Message* message = self->message; - const google::protobuf::Reflection* reflection = message->GetReflection(); + Message* message = self->message; + const Reflection* reflection = message->GetReflection(); for (Py_ssize_t i = child_length; i < message_length; ++i) { const Message& sub_message = reflection->GetRepeatedMessage( *(self->message), self->parent_field_descriptor, i); @@ -233,7 +231,7 @@ static int UpdateChildMessages(RepeatedCompositeContainer* self) { return -1; } cmsg->owner = self->owner; - cmsg->message = const_cast<google::protobuf::Message*>(&sub_message); + cmsg->message = const_cast<Message*>(&sub_message); cmsg->parent = self->parent; if (PyList_Append(self->child_messages, py_cmsg) < 0) { return -1; @@ -255,8 +253,8 @@ static PyObject* AddToAttached(RepeatedCompositeContainer* self, } if (cmessage::AssureWritable(self->parent) == -1) return NULL; - google::protobuf::Message* message = self->message; - google::protobuf::Message* sub_message = + Message* message = self->message; + Message* sub_message = message->GetReflection()->AddMessage(message, self->parent_field_descriptor); CMessage* cmsg = cmessage::NewEmptyMessage(self->subclass_init, @@ -482,9 +480,9 @@ static PyObject* SortAttached(RepeatedCompositeContainer* self, // Finally reverse the result if requested. if (reverse) { - google::protobuf::Message* message = self->message; - const google::protobuf::Reflection* reflection = message->GetReflection(); - const google::protobuf::FieldDescriptor* descriptor = self->parent_field_descriptor; + Message* message = self->message; + const Reflection* reflection = message->GetReflection(); + const FieldDescriptor* descriptor = self->parent_field_descriptor; // Reverse the Message array. for (int i = 0; i < length / 2; ++i) @@ -554,6 +552,26 @@ static PyObject* Item(RepeatedCompositeContainer* self, Py_ssize_t index) { return item; } +static PyObject* Pop(RepeatedCompositeContainer* self, + PyObject* args) { + Py_ssize_t index = -1; + if (!PyArg_ParseTuple(args, "|n", &index)) { + return NULL; + } + PyObject* item = Item(self, index); + if (item == NULL) { + PyErr_Format(PyExc_IndexError, + "list index (%zd) out of range", + index); + return NULL; + } + ScopedPyObjectPtr py_index(PyLong_FromSsize_t(index)); + if (AssignSubscript(self, py_index, NULL) < 0) { + return NULL; + } + return item; +} + // The caller takes ownership of the returned Message. Message* ReleaseLast(const FieldDescriptor* field, const Descriptor* type, @@ -571,7 +589,8 @@ Message* ReleaseLast(const FieldDescriptor* field, // the latter points to the default instance via a const_cast<>, so we // have to reset it to a new mutable object since we are taking ownership. if (released_message == NULL) { - const Message* prototype = global_message_factory->GetPrototype(type); + const Message* prototype = + cmessage::GetMessageFactory()->GetPrototype(type); GOOGLE_CHECK_NOTNULL(prototype); return prototype->New(); } else { @@ -646,7 +665,7 @@ int SetOwner(RepeatedCompositeContainer* self, // The private constructor of RepeatedCompositeContainer objects. PyObject *NewContainer( CMessage* parent, - const google::protobuf::FieldDescriptor* parent_field_descriptor, + const FieldDescriptor* parent_field_descriptor, PyObject *concrete_class) { if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) { return NULL; @@ -698,6 +717,8 @@ static PyMethodDef Methods[] = { "Adds an object to the repeated container." }, { "extend", (PyCFunction) Extend, METH_O, "Adds objects to the repeated container." }, + { "pop", (PyCFunction)Pop, METH_VARARGS, + "Removes an object from the repeated container and returns it." }, { "remove", (PyCFunction) Remove, METH_O, "Removes an object from the repeated container." }, { "sort", (PyCFunction) Sort, METH_VARARGS | METH_KEYWORDS, @@ -711,8 +732,8 @@ static PyMethodDef Methods[] = { PyTypeObject RepeatedCompositeContainer_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - "google.protobuf.internal." - "cpp._message.RepeatedCompositeContainer", // tp_name + "google.protobuf.pyext." + "_message.RepeatedCompositeContainer", // tp_name sizeof(RepeatedCompositeContainer), // tp_basicsize 0, // tp_itemsize (destructor)repeated_composite_container::Dealloc, // tp_dealloc diff --git a/python/google/protobuf/pyext/repeated_composite_container.h b/python/google/protobuf/pyext/repeated_composite_container.h index 0969af08..ce7cee0f 100644 --- a/python/google/protobuf/pyext/repeated_composite_container.h +++ b/python/google/protobuf/pyext/repeated_composite_container.h @@ -43,7 +43,6 @@ #include <string> #include <vector> - namespace google { namespace protobuf { @@ -82,7 +81,7 @@ typedef struct RepeatedCompositeContainer { // A descriptor used to modify the underlying 'message'. // The pointer is owned by the global DescriptorPool. - const google::protobuf::FieldDescriptor* parent_field_descriptor; + const FieldDescriptor* parent_field_descriptor; // Pointer to the C++ Message that contains this container. The // RepeatedCompositeContainer does not own this pointer. @@ -106,7 +105,7 @@ namespace repeated_composite_container { // field descriptor. PyObject *NewContainer( CMessage* parent, - const google::protobuf::FieldDescriptor* parent_field_descriptor, + const FieldDescriptor* parent_field_descriptor, PyObject *concrete_class); // Returns the number of items in this repeated composite container. @@ -150,8 +149,7 @@ int AssignSubscript(RepeatedCompositeContainer* self, // Releases the messages in the container to the given message. // // Returns 0 on success, -1 on failure. -int ReleaseToMessage(RepeatedCompositeContainer* self, - google::protobuf::Message* new_message); +int ReleaseToMessage(RepeatedCompositeContainer* self, Message* new_message); // Releases the messages in the container to a new message. // diff --git a/python/google/protobuf/pyext/repeated_scalar_container.cc b/python/google/protobuf/pyext/repeated_scalar_container.cc index 49d23fd6..17474598 100644 --- a/python/google/protobuf/pyext/repeated_scalar_container.cc +++ b/python/google/protobuf/pyext/repeated_scalar_container.cc @@ -60,8 +60,6 @@ namespace google { namespace protobuf { namespace python { -extern google::protobuf::DynamicMessageFactory* global_message_factory; - namespace repeated_scalar_container { static int InternalAssignRepeatedField( @@ -78,7 +76,7 @@ static int InternalAssignRepeatedField( } static Py_ssize_t Len(RepeatedScalarContainer* self) { - google::protobuf::Message* message = self->message; + Message* message = self->message; return message->GetReflection()->FieldSize(*message, self->parent_field_descriptor); } @@ -87,11 +85,10 @@ static int AssignItem(RepeatedScalarContainer* self, Py_ssize_t index, PyObject* arg) { cmessage::AssureWritable(self->parent); - google::protobuf::Message* message = self->message; - const google::protobuf::FieldDescriptor* field_descriptor = - self->parent_field_descriptor; + Message* message = self->message; + const FieldDescriptor* field_descriptor = self->parent_field_descriptor; - const google::protobuf::Reflection* reflection = message->GetReflection(); + const Reflection* reflection = message->GetReflection(); int field_size = reflection->FieldSize(*message, field_descriptor); if (index < 0) { index = field_size + index; @@ -115,64 +112,68 @@ static int AssignItem(RepeatedScalarContainer* self, } switch (field_descriptor->cpp_type()) { - case google::protobuf::FieldDescriptor::CPPTYPE_INT32: { + case FieldDescriptor::CPPTYPE_INT32: { GOOGLE_CHECK_GET_INT32(arg, value, -1); reflection->SetRepeatedInt32(message, field_descriptor, index, value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_INT64: { + case FieldDescriptor::CPPTYPE_INT64: { GOOGLE_CHECK_GET_INT64(arg, value, -1); reflection->SetRepeatedInt64(message, field_descriptor, index, value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: { + case FieldDescriptor::CPPTYPE_UINT32: { GOOGLE_CHECK_GET_UINT32(arg, value, -1); reflection->SetRepeatedUInt32(message, field_descriptor, index, value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: { + case FieldDescriptor::CPPTYPE_UINT64: { GOOGLE_CHECK_GET_UINT64(arg, value, -1); reflection->SetRepeatedUInt64(message, field_descriptor, index, value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: { + case FieldDescriptor::CPPTYPE_FLOAT: { GOOGLE_CHECK_GET_FLOAT(arg, value, -1); reflection->SetRepeatedFloat(message, field_descriptor, index, value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: { + case FieldDescriptor::CPPTYPE_DOUBLE: { GOOGLE_CHECK_GET_DOUBLE(arg, value, -1); reflection->SetRepeatedDouble(message, field_descriptor, index, value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: { + case FieldDescriptor::CPPTYPE_BOOL: { GOOGLE_CHECK_GET_BOOL(arg, value, -1); reflection->SetRepeatedBool(message, field_descriptor, index, value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_STRING: { + case FieldDescriptor::CPPTYPE_STRING: { if (!CheckAndSetString( arg, message, field_descriptor, reflection, false, index)) { return -1; } break; } - case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: { + case FieldDescriptor::CPPTYPE_ENUM: { GOOGLE_CHECK_GET_INT32(arg, value, -1); - const google::protobuf::EnumDescriptor* enum_descriptor = - field_descriptor->enum_type(); - const google::protobuf::EnumValueDescriptor* enum_value = - enum_descriptor->FindValueByNumber(value); - if (enum_value != NULL) { - reflection->SetRepeatedEnum(message, field_descriptor, index, - enum_value); + if (reflection->SupportsUnknownEnumValues()) { + reflection->SetRepeatedEnumValue(message, field_descriptor, index, + value); } else { - ScopedPyObjectPtr s(PyObject_Str(arg)); - if (s != NULL) { - PyErr_Format(PyExc_ValueError, "Unknown enum value: %s", - PyString_AsString(s)); + const EnumDescriptor* enum_descriptor = field_descriptor->enum_type(); + const EnumValueDescriptor* enum_value = + enum_descriptor->FindValueByNumber(value); + if (enum_value != NULL) { + reflection->SetRepeatedEnum(message, field_descriptor, index, + enum_value); + } else { + ScopedPyObjectPtr s(PyObject_Str(arg)); + if (s != NULL) { + PyErr_Format(PyExc_ValueError, "Unknown enum value: %s", + PyString_AsString(s)); + } + return -1; } - return -1; } break; } @@ -186,10 +187,9 @@ static int AssignItem(RepeatedScalarContainer* self, } static PyObject* Item(RepeatedScalarContainer* self, Py_ssize_t index) { - google::protobuf::Message* message = self->message; - const google::protobuf::FieldDescriptor* field_descriptor = - self->parent_field_descriptor; - const google::protobuf::Reflection* reflection = message->GetReflection(); + Message* message = self->message; + const FieldDescriptor* field_descriptor = self->parent_field_descriptor; + const Reflection* reflection = message->GetReflection(); int field_size = reflection->FieldSize(*message, field_descriptor); if (index < 0) { @@ -197,80 +197,80 @@ static PyObject* Item(RepeatedScalarContainer* self, Py_ssize_t index) { } if (index < 0 || index >= field_size) { PyErr_Format(PyExc_IndexError, - "list assignment index (%d) out of range", - static_cast<int>(index)); + "list index (%zd) out of range", + index); return NULL; } PyObject* result = NULL; switch (field_descriptor->cpp_type()) { - case google::protobuf::FieldDescriptor::CPPTYPE_INT32: { + case FieldDescriptor::CPPTYPE_INT32: { int32 value = reflection->GetRepeatedInt32( *message, field_descriptor, index); result = PyInt_FromLong(value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_INT64: { + case FieldDescriptor::CPPTYPE_INT64: { int64 value = reflection->GetRepeatedInt64( *message, field_descriptor, index); result = PyLong_FromLongLong(value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: { + case FieldDescriptor::CPPTYPE_UINT32: { uint32 value = reflection->GetRepeatedUInt32( *message, field_descriptor, index); result = PyLong_FromLongLong(value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: { + case FieldDescriptor::CPPTYPE_UINT64: { uint64 value = reflection->GetRepeatedUInt64( *message, field_descriptor, index); result = PyLong_FromUnsignedLongLong(value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: { + case FieldDescriptor::CPPTYPE_FLOAT: { float value = reflection->GetRepeatedFloat( *message, field_descriptor, index); result = PyFloat_FromDouble(value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: { + case FieldDescriptor::CPPTYPE_DOUBLE: { double value = reflection->GetRepeatedDouble( *message, field_descriptor, index); result = PyFloat_FromDouble(value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: { + case FieldDescriptor::CPPTYPE_BOOL: { bool value = reflection->GetRepeatedBool( *message, field_descriptor, index); result = PyBool_FromLong(value ? 1 : 0); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: { - const google::protobuf::EnumValueDescriptor* enum_value = + case FieldDescriptor::CPPTYPE_ENUM: { + const EnumValueDescriptor* enum_value = message->GetReflection()->GetRepeatedEnum( *message, field_descriptor, index); result = PyInt_FromLong(enum_value->number()); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_STRING: { + case FieldDescriptor::CPPTYPE_STRING: { string value = reflection->GetRepeatedString( *message, field_descriptor, index); result = ToStringObject(field_descriptor, value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { + case FieldDescriptor::CPPTYPE_MESSAGE: { PyObject* py_cmsg = PyObject_CallObject(reinterpret_cast<PyObject*>( &CMessage_Type), NULL); if (py_cmsg == NULL) { return NULL; } CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg); - const google::protobuf::Message& msg = reflection->GetRepeatedMessage( + const Message& msg = reflection->GetRepeatedMessage( *message, field_descriptor, index); cmsg->owner = self->owner; cmsg->parent = self->parent; - cmsg->message = const_cast<google::protobuf::Message*>(&msg); + cmsg->message = const_cast<Message*>(&msg); cmsg->read_only = false; result = reinterpret_cast<PyObject*>(py_cmsg); break; @@ -351,69 +351,71 @@ static PyObject* Subscript(RepeatedScalarContainer* self, PyObject* slice) { PyObject* Append(RepeatedScalarContainer* self, PyObject* item) { cmessage::AssureWritable(self->parent); - google::protobuf::Message* message = self->message; - const google::protobuf::FieldDescriptor* field_descriptor = - self->parent_field_descriptor; + Message* message = self->message; + const FieldDescriptor* field_descriptor = self->parent_field_descriptor; - const google::protobuf::Reflection* reflection = message->GetReflection(); + const Reflection* reflection = message->GetReflection(); switch (field_descriptor->cpp_type()) { - case google::protobuf::FieldDescriptor::CPPTYPE_INT32: { + case FieldDescriptor::CPPTYPE_INT32: { GOOGLE_CHECK_GET_INT32(item, value, NULL); reflection->AddInt32(message, field_descriptor, value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_INT64: { + case FieldDescriptor::CPPTYPE_INT64: { GOOGLE_CHECK_GET_INT64(item, value, NULL); reflection->AddInt64(message, field_descriptor, value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: { + case FieldDescriptor::CPPTYPE_UINT32: { GOOGLE_CHECK_GET_UINT32(item, value, NULL); reflection->AddUInt32(message, field_descriptor, value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: { + case FieldDescriptor::CPPTYPE_UINT64: { GOOGLE_CHECK_GET_UINT64(item, value, NULL); reflection->AddUInt64(message, field_descriptor, value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: { + case FieldDescriptor::CPPTYPE_FLOAT: { GOOGLE_CHECK_GET_FLOAT(item, value, NULL); reflection->AddFloat(message, field_descriptor, value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: { + case FieldDescriptor::CPPTYPE_DOUBLE: { GOOGLE_CHECK_GET_DOUBLE(item, value, NULL); reflection->AddDouble(message, field_descriptor, value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: { + case FieldDescriptor::CPPTYPE_BOOL: { GOOGLE_CHECK_GET_BOOL(item, value, NULL); reflection->AddBool(message, field_descriptor, value); break; } - case google::protobuf::FieldDescriptor::CPPTYPE_STRING: { + case FieldDescriptor::CPPTYPE_STRING: { if (!CheckAndSetString( item, message, field_descriptor, reflection, true, -1)) { return NULL; } break; } - case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: { + case FieldDescriptor::CPPTYPE_ENUM: { GOOGLE_CHECK_GET_INT32(item, value, NULL); - const google::protobuf::EnumDescriptor* enum_descriptor = - field_descriptor->enum_type(); - const google::protobuf::EnumValueDescriptor* enum_value = - enum_descriptor->FindValueByNumber(value); - if (enum_value != NULL) { - reflection->AddEnum(message, field_descriptor, enum_value); + if (reflection->SupportsUnknownEnumValues()) { + reflection->AddEnumValue(message, field_descriptor, value); } else { - ScopedPyObjectPtr s(PyObject_Str(item)); - if (s != NULL) { - PyErr_Format(PyExc_ValueError, "Unknown enum value: %s", - PyString_AsString(s)); + const EnumDescriptor* enum_descriptor = field_descriptor->enum_type(); + const EnumValueDescriptor* enum_value = + enum_descriptor->FindValueByNumber(value); + if (enum_value != NULL) { + reflection->AddEnum(message, field_descriptor, enum_value); + } else { + ScopedPyObjectPtr s(PyObject_Str(item)); + if (s != NULL) { + PyErr_Format(PyExc_ValueError, "Unknown enum value: %s", + PyString_AsString(s)); + } + return NULL; } - return NULL; } break; } @@ -438,8 +440,8 @@ static int AssSubscript(RepeatedScalarContainer* self, bool create_list = false; cmessage::AssureWritable(self->parent); - google::protobuf::Message* message = self->message; - const google::protobuf::FieldDescriptor* field_descriptor = + Message* message = self->message; + const FieldDescriptor* field_descriptor = self->parent_field_descriptor; #if PY_MAJOR_VERSION < 3 @@ -450,7 +452,7 @@ static int AssSubscript(RepeatedScalarContainer* self, if (PyLong_Check(slice)) { from = to = PyLong_AsLong(slice); } else if (PySlice_Check(slice)) { - const google::protobuf::Reflection* reflection = message->GetReflection(); + const Reflection* reflection = message->GetReflection(); length = reflection->FieldSize(*message, field_descriptor); #if PY_MAJOR_VERSION >= 3 if (PySlice_GetIndicesEx(slice, @@ -492,9 +494,15 @@ static int AssSubscript(RepeatedScalarContainer* self, PyObject* Extend(RepeatedScalarContainer* self, PyObject* value) { cmessage::AssureWritable(self->parent); - if (PyObject_Not(value)) { + + // TODO(ptucker): Deprecate this behavior. b/18413862 + if (value == Py_None) { + Py_RETURN_NONE; + } + if ((Py_TYPE(value)->tp_as_sequence == NULL) && PyObject_Not(value)) { Py_RETURN_NONE; } + ScopedPyObjectPtr iter(PyObject_GetIter(value)); if (iter == NULL) { PyErr_SetString(PyExc_TypeError, "Value must be iterable"); @@ -627,9 +635,28 @@ static PyObject* Sort(RepeatedScalarContainer* self, Py_RETURN_NONE; } +static PyObject* Pop(RepeatedScalarContainer* self, + PyObject* args) { + Py_ssize_t index = -1; + if (!PyArg_ParseTuple(args, "|n", &index)) { + return NULL; + } + PyObject* item = Item(self, index); + if (item == NULL) { + PyErr_Format(PyExc_IndexError, + "list index (%zd) out of range", + index); + return NULL; + } + if (AssignItem(self, index, NULL) < 0) { + return NULL; + } + return item; +} + // The private constructor of RepeatedScalarContainer objects. PyObject *NewContainer( - CMessage* parent, const google::protobuf::FieldDescriptor* parent_field_descriptor) { + CMessage* parent, const FieldDescriptor* parent_field_descriptor) { if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) { return NULL; } @@ -663,7 +690,7 @@ static int InitializeAndCopyToParentContainer( if (values == NULL) { return -1; } - google::protobuf::Message* new_message = global_message_factory->GetPrototype( + Message* new_message = cmessage::GetMessageFactory()->GetPrototype( from->message->GetDescriptor())->New(); to->parent = NULL; to->parent_field_descriptor = from->parent_field_descriptor; @@ -729,6 +756,8 @@ static PyMethodDef Methods[] = { "Appends objects to the repeated container." }, { "insert", (PyCFunction)Insert, METH_VARARGS, "Appends objects to the repeated container." }, + { "pop", (PyCFunction)Pop, METH_VARARGS, + "Removes an object from the repeated container and returns it." }, { "remove", (PyCFunction)Remove, METH_O, "Removes an object from the repeated container." }, { "sort", (PyCFunction)Sort, METH_VARARGS | METH_KEYWORDS, @@ -740,8 +769,8 @@ static PyMethodDef Methods[] = { PyTypeObject RepeatedScalarContainer_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - "google.protobuf.internal." - "cpp._message.RepeatedScalarContainer", // tp_name + "google.protobuf." + "pyext._message.RepeatedScalarContainer", // tp_name sizeof(RepeatedScalarContainer), // tp_basicsize 0, // tp_itemsize (destructor)repeated_scalar_container::Dealloc, // tp_dealloc diff --git a/python/google/protobuf/pyext/repeated_scalar_container.h b/python/google/protobuf/pyext/repeated_scalar_container.h index 513bfe48..5dfa21e0 100644 --- a/python/google/protobuf/pyext/repeated_scalar_container.h +++ b/python/google/protobuf/pyext/repeated_scalar_container.h @@ -77,7 +77,7 @@ typedef struct RepeatedScalarContainer { // field. Used together with the parent's message when making a // default message instance mutable. // The pointer is owned by the global DescriptorPool. - const google::protobuf::FieldDescriptor* parent_field_descriptor; + const FieldDescriptor* parent_field_descriptor; } RepeatedScalarContainer; extern PyTypeObject RepeatedScalarContainer_Type; @@ -87,7 +87,7 @@ namespace repeated_scalar_container { // Builds a RepeatedScalarContainer object, from a parent message and a // field descriptor. extern PyObject *NewContainer( - CMessage* parent, const google::protobuf::FieldDescriptor* parent_field_descriptor); + CMessage* parent, const FieldDescriptor* parent_field_descriptor); // Appends the scalar 'item' to the end of the container 'self'. // diff --git a/python/google/protobuf/reflection.py b/python/google/protobuf/reflection.py index 1fc704a2..55e653a0 100755 --- a/python/google/protobuf/reflection.py +++ b/python/google/protobuf/reflection.py @@ -56,18 +56,12 @@ _FieldDescriptor = descriptor_mod.FieldDescriptor if api_implementation.Type() == 'cpp': - if api_implementation.Version() == 2: - from google.protobuf.pyext import cpp_message - _NewMessage = cpp_message.NewMessage - _InitMessage = cpp_message.InitMessage - else: - from google.protobuf.internal import cpp_message - _NewMessage = cpp_message.NewMessage - _InitMessage = cpp_message.InitMessage + from google.protobuf.pyext import cpp_message as message_impl else: - from google.protobuf.internal import python_message - _NewMessage = python_message.NewMessage - _InitMessage = python_message.InitMessage + from google.protobuf.internal import python_message as message_impl + +_NewMessage = message_impl.NewMessage +_InitMessage = message_impl.InitMessage class GeneratedProtocolMessageType(type): @@ -127,7 +121,6 @@ class GeneratedProtocolMessageType(type): superclass = super(GeneratedProtocolMessageType, cls) new_class = superclass.__new__(cls, name, bases, dictionary) - setattr(descriptor, '_concrete_class', new_class) return new_class def __init__(cls, name, bases, dictionary): @@ -151,6 +144,7 @@ class GeneratedProtocolMessageType(type): _InitMessage(descriptor, cls) superclass = super(GeneratedProtocolMessageType, cls) superclass.__init__(name, bases, dictionary) + setattr(descriptor, '_concrete_class', cls) def ParseMessage(descriptor, byte_str): diff --git a/python/google/protobuf/text_format.py b/python/google/protobuf/text_format.py index fb54c50c..a47ce3e3 100755 --- a/python/google/protobuf/text_format.py +++ b/python/google/protobuf/text_format.py @@ -319,6 +319,11 @@ def _MergeField(tokenizer, message, allow_multiple_scalars): ParseError: In case of ASCII parsing problems. """ message_descriptor = message.DESCRIPTOR + if (hasattr(message_descriptor, 'syntax') and + message_descriptor.syntax == 'proto3'): + # Proto3 doesn't represent presence so we can't test if multiple + # scalars have occurred. We have to allow them. + allow_multiple_scalars = True if tokenizer.TryConsume('['): name = [tokenizer.ConsumeIdentifier()] while tokenizer.TryConsume('.'): diff --git a/python/setup.py b/python/setup.py index cfe25cc0..c6ff7454 100755 --- a/python/setup.py +++ b/python/setup.py @@ -77,6 +77,7 @@ def GenerateUnittestProtos(): generate_proto("../src/google/protobuf/unittest_import_public.proto") generate_proto("../src/google/protobuf/unittest_mset.proto") generate_proto("../src/google/protobuf/unittest_no_generic_services.proto") + generate_proto("../src/google/protobuf/unittest_proto3_arena.proto") generate_proto("google/protobuf/internal/descriptor_pool_test1.proto") generate_proto("google/protobuf/internal/descriptor_pool_test2.proto") generate_proto("google/protobuf/internal/test_bad_identifiers.proto") @@ -90,23 +91,6 @@ def GenerateUnittestProtos(): generate_proto("google/protobuf/internal/import_test_package/outer.proto") generate_proto("google/protobuf/pyext/python.proto") -def MakeTestSuite(): - # Test C++ implementation - import unittest - import google.protobuf.pyext.descriptor_cpp2_test as descriptor_cpp2_test - import google.protobuf.pyext.message_factory_cpp2_test \ - as message_factory_cpp2_test - import google.protobuf.pyext.reflection_cpp2_generated_test \ - as reflection_cpp2_generated_test - - loader = unittest.defaultTestLoader - suite = unittest.TestSuite() - for test in [ descriptor_cpp2_test, - message_factory_cpp2_test, - reflection_cpp2_generated_test]: - suite.addTest(loader.loadTestsFromModule(test)) - return suite - class clean(_clean): def run(self): # Delete generated files in the code tree. @@ -152,6 +136,8 @@ if __name__ == '__main__': ext_module_list.append(Extension( "google.protobuf.pyext._message", [ "google/protobuf/pyext/descriptor.cc", + "google/protobuf/pyext/descriptor_containers.cc", + "google/protobuf/pyext/descriptor_pool.cc", "google/protobuf/pyext/message.cc", "google/protobuf/pyext/extension_dict.cc", "google/protobuf/pyext/repeated_scalar_container.cc", @@ -161,12 +147,12 @@ if __name__ == '__main__': libraries = [ "protobuf" ], library_dirs = [ '../src/.libs' ], )) + os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp' setup(name = 'protobuf', - version = '3.0.0-pre', + version = '3.0.0-alpha-2', packages = [ 'google' ], namespace_packages = [ 'google' ], - test_suite = 'setup.MakeTestSuite', google_test_dir = "google/protobuf/internal", # Must list modules explicitly so that we don't install tests. py_modules = [ diff --git a/ruby/README.md b/ruby/README.md index c966a103..84f4a775 100644 --- a/ruby/README.md +++ b/ruby/README.md @@ -7,8 +7,51 @@ we recommend using protoc's Ruby generation support with .proto files. The build process in this directory only installs the extension; you need to install protoc as well to have Ruby code generation functionality. -Installation ------------- +Installation from Gem +--------------------- + +When we release a version of Protocol Buffers, we will upload a Gem to +[RubyGems](https://www.rubygems.org/). To use this pre-packaged gem, simply +install it as you would any other gem: + + $ gem install [--prerelease] google-protobuf + +The `--pre` flag is necessary if we have not yet made a non-alpha/beta release +of the Ruby extension; it allows `gem` to consider these "pre-release" +alpha/beta versions. + +Once the gem is installed, you may or may not need `protoc`. If you write your +message type descriptions directly in the Ruby DSL, you do not need it. +However, if you wish to generate the Ruby DSL from a `.proto` file, you will +also want to install Protocol Buffers itself, as described in this repository's +main `README` file. The version of `protoc` included in the latest release +supports the `--ruby_out` option to generate Ruby code. + +A simple example of using the Ruby extension follows. More extensive +documentation may be found in the RubyDoc comments (`call-seq` tags) in the +source, and we plan to release separate, more detailed, documentation at a +later date. + + require 'google/protobuf' + + # generated from my_proto_types.proto with protoc: + # $ protoc --ruby_out=. my_proto_types.proto + require 'my_proto_types' + + mymessage = MyTestMessage.new(:field1 => 42, :field2 => ["a", "b", "c"]) + mymessage.field1 = 43 + mymessage.field2.push("d") + mymessage.field3 = SubMessage.new(:foo => 100) + + encoded_data = MyTestMessage.encode(mymessage) + decoded = MyTestMessage.decode(encoded_data) + assert decoded == mymessage + + puts "JSON:" + puts MyTestMessage.encode_json(mymessage) + +Installation from Source (Building Gem) +--------------------------------------- To build this Ruby extension, you will need: @@ -16,15 +59,8 @@ To build this Ruby extension, you will need: * Bundler * Ruby development headers * a C compiler -* the upb submodule -First, ensure that upb/ is checked out: - - $ cd .. # top level protobuf directory - $ git submodule init - $ git submodule update - -Then install the required Ruby gems: +First, install the required Ruby gems: $ sudo gem install bundler rake rake-compiler rspec rubygems-tasks @@ -32,3 +68,7 @@ Then build the Gem: $ rake gem $ gem install pkg/protobuf-$VERSION.gem + +This gem includes the upb parsing and serialization library as a single-file +amalgamation. It is up-to-date with upb git commit +`535bc2fe2f2b467f59347ffc9449e11e47791257`. diff --git a/ruby/ext/google/protobuf_c/defs.c b/ruby/ext/google/protobuf_c/defs.c index a18aaac4..b39c27f4 100644 --- a/ruby/ext/google/protobuf_c/defs.c +++ b/ruby/ext/google/protobuf_c/defs.c @@ -58,11 +58,15 @@ static upb_def* check_notfrozen(const upb_def* def) { } static upb_msgdef* check_msg_notfrozen(const upb_msgdef* def) { - return (upb_msgdef*)check_notfrozen((const upb_def*)def); + return upb_downcast_msgdef_mutable(check_notfrozen((const upb_def*)def)); } static upb_fielddef* check_field_notfrozen(const upb_fielddef* def) { - return (upb_fielddef*)check_notfrozen((const upb_def*)def); + return upb_downcast_fielddef_mutable(check_notfrozen((const upb_def*)def)); +} + +static upb_oneofdef* check_oneof_notfrozen(const upb_oneofdef* def) { + return (upb_oneofdef*)check_notfrozen((const upb_def*)def); } static upb_enumdef* check_enum_notfrozen(const upb_enumdef* def) { @@ -282,6 +286,9 @@ void Descriptor_register(VALUE module) { rb_define_method(klass, "each", Descriptor_each, 0); rb_define_method(klass, "lookup", Descriptor_lookup, 1); rb_define_method(klass, "add_field", Descriptor_add_field, 1); + rb_define_method(klass, "add_oneof", Descriptor_add_oneof, 1); + rb_define_method(klass, "each_oneof", Descriptor_each_oneof, 0); + rb_define_method(klass, "lookup_oneof", Descriptor_lookup_oneof, 1); rb_define_method(klass, "msgclass", Descriptor_msgclass, 0); rb_define_method(klass, "name", Descriptor_name, 0); rb_define_method(klass, "name=", Descriptor_name_set, 1); @@ -328,10 +335,10 @@ VALUE Descriptor_name_set(VALUE _self, VALUE str) { VALUE Descriptor_each(VALUE _self) { DEFINE_SELF(Descriptor, self, _self); - upb_msg_iter it; - for (upb_msg_begin(&it, self->msgdef); - !upb_msg_done(&it); - upb_msg_next(&it)) { + upb_msg_field_iter it; + for (upb_msg_field_begin(&it, self->msgdef); + !upb_msg_field_done(&it); + upb_msg_field_next(&it)) { const upb_fielddef* field = upb_msg_iter_field(&it); VALUE obj = get_def_obj(field); rb_yield(obj); @@ -360,7 +367,7 @@ VALUE Descriptor_lookup(VALUE _self, VALUE name) { * call-seq: * Descriptor.add_field(field) => nil * - * Adds the given FieldDescriptor to this message type. The descriptor must not + * Adds the given FieldDescriptor to this message type. This descriptor must not * have been added to a pool yet. Raises an exception if a field with the same * name or number already exists. Sub-type references (e.g. for fields of type * message) are not resolved at this point. @@ -379,6 +386,67 @@ VALUE Descriptor_add_field(VALUE _self, VALUE obj) { /* * call-seq: + * Descriptor.add_oneof(oneof) => nil + * + * Adds the given OneofDescriptor to this message type. This descriptor must not + * have been added to a pool yet. Raises an exception if a oneof with the same + * name already exists, or if any of the oneof's fields' names or numbers + * conflict with an existing field in this message type. All fields in the oneof + * are added to the message descriptor. Sub-type references (e.g. for fields of + * type message) are not resolved at this point. + */ +VALUE Descriptor_add_oneof(VALUE _self, VALUE obj) { + DEFINE_SELF(Descriptor, self, _self); + upb_msgdef* mut_def = check_msg_notfrozen(self->msgdef); + OneofDescriptor* def = ruby_to_OneofDescriptor(obj); + upb_oneofdef* mut_oneof_def = check_oneof_notfrozen(def->oneofdef); + CHECK_UPB( + upb_msgdef_addoneof(mut_def, mut_oneof_def, NULL, &status), + "Adding oneof to Descriptor failed"); + add_def_obj(def->oneofdef, obj); + return Qnil; +} + +/* + * call-seq: + * Descriptor.each_oneof(&block) => nil + * + * Invokes the given block for each oneof in this message type, passing the + * corresponding OneofDescriptor. + */ +VALUE Descriptor_each_oneof(VALUE _self) { + DEFINE_SELF(Descriptor, self, _self); + + upb_msg_oneof_iter it; + for (upb_msg_oneof_begin(&it, self->msgdef); + !upb_msg_oneof_done(&it); + upb_msg_oneof_next(&it)) { + const upb_oneofdef* oneof = upb_msg_iter_oneof(&it); + VALUE obj = get_def_obj(oneof); + rb_yield(obj); + } + return Qnil; +} + +/* + * call-seq: + * Descriptor.lookup_oneof(name) => OneofDescriptor + * + * Returns the oneof descriptor for the oneof with the given name, if present, + * or nil if none. + */ +VALUE Descriptor_lookup_oneof(VALUE _self, VALUE name) { + DEFINE_SELF(Descriptor, self, _self); + const char* s = get_str(name); + const upb_oneofdef* oneof = upb_msgdef_ntooz(self->msgdef, s); + if (oneof == NULL) { + return Qnil; + } + return get_def_obj(oneof); +} + +/* + * call-seq: * Descriptor.msgclass => message_klass * * Returns the Ruby class created for this message type. Valid only once the @@ -744,6 +812,120 @@ VALUE FieldDescriptor_set(VALUE _self, VALUE msg_rb, VALUE value) { } // ----------------------------------------------------------------------------- +// OneofDescriptor. +// ----------------------------------------------------------------------------- + +DEFINE_CLASS(OneofDescriptor, "Google::Protobuf::OneofDescriptor"); + +void OneofDescriptor_mark(void* _self) { +} + +void OneofDescriptor_free(void* _self) { + OneofDescriptor* self = _self; + upb_oneofdef_unref(self->oneofdef, &self->oneofdef); + xfree(self); +} + +/* + * call-seq: + * OneofDescriptor.new => oneof_descriptor + * + * Creates a new, empty, oneof descriptor. The oneof may only be modified prior + * to being added to a message descriptor which is subsequently added to a pool. + */ +VALUE OneofDescriptor_alloc(VALUE klass) { + OneofDescriptor* self = ALLOC(OneofDescriptor); + VALUE ret = TypedData_Wrap_Struct(klass, &_OneofDescriptor_type, self); + self->oneofdef = upb_oneofdef_new(&self->oneofdef); + return ret; +} + +void OneofDescriptor_register(VALUE module) { + VALUE klass = rb_define_class_under( + module, "OneofDescriptor", rb_cObject); + rb_define_alloc_func(klass, OneofDescriptor_alloc); + rb_define_method(klass, "name", OneofDescriptor_name, 0); + rb_define_method(klass, "name=", OneofDescriptor_name_set, 1); + rb_define_method(klass, "add_field", OneofDescriptor_add_field, 1); + rb_define_method(klass, "each", OneofDescriptor_each, 0); + rb_include_module(klass, rb_mEnumerable); + cOneofDescriptor = klass; + rb_gc_register_address(&cOneofDescriptor); +} + +/* + * call-seq: + * OneofDescriptor.name => name + * + * Returns the name of this oneof. + */ +VALUE OneofDescriptor_name(VALUE _self) { + DEFINE_SELF(OneofDescriptor, self, _self); + return rb_str_maybe_null(upb_oneofdef_name(self->oneofdef)); +} + +/* + * call-seq: + * OneofDescriptor.name = name + * + * Sets a new name for this oneof. The oneof must not have been added to a + * message descriptor yet. + */ +VALUE OneofDescriptor_name_set(VALUE _self, VALUE value) { + DEFINE_SELF(OneofDescriptor, self, _self); + upb_oneofdef* mut_def = check_oneof_notfrozen(self->oneofdef); + const char* str = get_str(value); + CHECK_UPB(upb_oneofdef_setname(mut_def, str, &status), + "Error setting oneof name"); + return Qnil; +} + +/* + * call-seq: + * OneofDescriptor.add_field(field) => nil + * + * Adds a field to this oneof. The field may have been added to this oneof in + * the past, or the message to which this oneof belongs (if any), but may not + * have already been added to any other oneof or message. Otherwise, an + * exception is raised. + * + * All fields added to the oneof via this method will be automatically added to + * the message to which this oneof belongs, if it belongs to one currently, or + * else will be added to any message to which the oneof is later added at the + * time that it is added. + */ +VALUE OneofDescriptor_add_field(VALUE _self, VALUE obj) { + DEFINE_SELF(OneofDescriptor, self, _self); + upb_oneofdef* mut_def = check_oneof_notfrozen(self->oneofdef); + FieldDescriptor* def = ruby_to_FieldDescriptor(obj); + upb_fielddef* mut_field_def = check_field_notfrozen(def->fielddef); + CHECK_UPB( + upb_oneofdef_addfield(mut_def, mut_field_def, NULL, &status), + "Adding field to OneofDescriptor failed"); + add_def_obj(def->fielddef, obj); + return Qnil; +} + +/* + * call-seq: + * OneofDescriptor.each(&block) => nil + * + * Iterates through fields in this oneof, yielding to the block on each one. + */ +VALUE OneofDescriptor_each(VALUE _self, VALUE field) { + DEFINE_SELF(OneofDescriptor, self, _self); + upb_oneof_iter it; + for (upb_oneof_begin(&it, self->oneofdef); + !upb_oneof_done(&it); + upb_oneof_next(&it)) { + const upb_fielddef* f = upb_oneof_iter_field(&it); + VALUE obj = get_def_obj(f); + rb_yield(obj); + } + return Qnil; +} + +// ----------------------------------------------------------------------------- // EnumDescriptor. // ----------------------------------------------------------------------------- @@ -952,6 +1134,7 @@ void MessageBuilderContext_register(VALUE module) { rb_define_method(klass, "required", MessageBuilderContext_required, -1); rb_define_method(klass, "repeated", MessageBuilderContext_repeated, -1); rb_define_method(klass, "map", MessageBuilderContext_map, -1); + rb_define_method(klass, "oneof", MessageBuilderContext_oneof, 1); cMessageBuilderContext = klass; rb_gc_register_address(&cMessageBuilderContext); } @@ -1165,6 +1348,110 @@ VALUE MessageBuilderContext_map(int argc, VALUE* argv, VALUE _self) { return Qnil; } +/* + * call-seq: + * MessageBuilderContext.oneof(name, &block) => nil + * + * Creates a new OneofDescriptor with the given name, creates a + * OneofBuilderContext attached to that OneofDescriptor, evaluates the given + * block in the context of that OneofBuilderContext with #instance_eval, and + * then adds the oneof to the message. + * + * This is the recommended, idiomatic way to build oneof definitions. + */ +VALUE MessageBuilderContext_oneof(VALUE _self, VALUE name) { + DEFINE_SELF(MessageBuilderContext, self, _self); + VALUE oneofdef = rb_class_new_instance(0, NULL, cOneofDescriptor); + VALUE args[2] = { oneofdef, self->builder }; + VALUE ctx = rb_class_new_instance(2, args, cOneofBuilderContext); + VALUE block = rb_block_proc(); + VALUE name_str = rb_str_new2(rb_id2name(SYM2ID(name))); + rb_funcall(oneofdef, rb_intern("name="), 1, name_str); + rb_funcall_with_block(ctx, rb_intern("instance_eval"), 0, NULL, block); + Descriptor_add_oneof(self->descriptor, oneofdef); + + return Qnil; +} + +// ----------------------------------------------------------------------------- +// OneofBuilderContext. +// ----------------------------------------------------------------------------- + +DEFINE_CLASS(OneofBuilderContext, + "Google::Protobuf::Internal::OneofBuilderContext"); + +void OneofBuilderContext_mark(void* _self) { + OneofBuilderContext* self = _self; + rb_gc_mark(self->descriptor); + rb_gc_mark(self->builder); +} + +void OneofBuilderContext_free(void* _self) { + OneofBuilderContext* self = _self; + xfree(self); +} + +VALUE OneofBuilderContext_alloc(VALUE klass) { + OneofBuilderContext* self = ALLOC(OneofBuilderContext); + VALUE ret = TypedData_Wrap_Struct( + klass, &_OneofBuilderContext_type, self); + self->descriptor = Qnil; + self->builder = Qnil; + return ret; +} + +void OneofBuilderContext_register(VALUE module) { + VALUE klass = rb_define_class_under( + module, "OneofBuilderContext", rb_cObject); + rb_define_alloc_func(klass, OneofBuilderContext_alloc); + rb_define_method(klass, "initialize", + OneofBuilderContext_initialize, 2); + rb_define_method(klass, "optional", OneofBuilderContext_optional, -1); + cOneofBuilderContext = klass; + rb_gc_register_address(&cOneofBuilderContext); +} + +/* + * call-seq: + * OneofBuilderContext.new(desc, builder) => context + * + * Create a new oneof builder context around the given oneof descriptor and + * builder context. This class is intended to serve as a DSL context to be used + * with #instance_eval. + */ +VALUE OneofBuilderContext_initialize(VALUE _self, + VALUE oneofdef, + VALUE builder) { + DEFINE_SELF(OneofBuilderContext, self, _self); + self->descriptor = oneofdef; + self->builder = builder; + return Qnil; +} + +/* + * call-seq: + * OneofBuilderContext.optional(name, type, number, type_class = nil) + * + * Defines a new optional field in this oneof with the given type, tag number, + * and type class (for message and enum fields). The type must be a Ruby symbol + * (as accepted by FieldDescriptor#type=) and the type_class must be a string, + * if present (as accepted by FieldDescriptor#submsg_name=). + */ +VALUE OneofBuilderContext_optional(int argc, VALUE* argv, VALUE _self) { + DEFINE_SELF(OneofBuilderContext, self, _self); + + if (argc < 3) { + rb_raise(rb_eArgError, "Expected at least 3 arguments."); + } + VALUE name = argv[0]; + VALUE type = argv[1]; + VALUE number = argv[2]; + VALUE type_class = (argc > 3) ? argv[3] : Qnil; + + return msgdef_add_field(self->descriptor, "optional", + name, type, number, type_class); +} + // ----------------------------------------------------------------------------- // EnumBuilderContext. // ----------------------------------------------------------------------------- @@ -1322,8 +1609,10 @@ VALUE Builder_add_enum(VALUE _self, VALUE name) { static void validate_msgdef(const upb_msgdef* msgdef) { // Verify that no required fields exist. proto3 does not support these. - upb_msg_iter it; - for (upb_msg_begin(&it, msgdef); !upb_msg_done(&it); upb_msg_next(&it)) { + upb_msg_field_iter it; + for (upb_msg_field_begin(&it, msgdef); + !upb_msg_field_done(&it); + upb_msg_field_next(&it)) { const upb_fielddef* field = upb_msg_iter_field(&it); if (upb_fielddef_label(field) == UPB_LABEL_REQUIRED) { rb_raise(rb_eTypeError, "Required fields are unsupported in proto3."); diff --git a/ruby/ext/google/protobuf_c/encode_decode.c b/ruby/ext/google/protobuf_c/encode_decode.c index e5e1514b..5730504d 100644 --- a/ruby/ext/google/protobuf_c/encode_decode.c +++ b/ruby/ext/google/protobuf_c/encode_decode.c @@ -59,6 +59,36 @@ static const void *newsubmsghandlerdata(upb_handlers* h, uint32_t ofs, return hd; } +typedef struct { + size_t ofs; // union data slot + size_t case_ofs; // oneof_case field + uint32_t oneof_case_num; // oneof-case number to place in oneof_case field + const upb_msgdef *md; // msgdef, for oneof submessage handler +} oneof_handlerdata_t; + +static const void *newoneofhandlerdata(upb_handlers *h, + uint32_t ofs, + uint32_t case_ofs, + const upb_fielddef *f) { + oneof_handlerdata_t *hd = ALLOC(oneof_handlerdata_t); + hd->ofs = ofs; + hd->case_ofs = case_ofs; + // We reuse the field tag number as a oneof union discriminant tag. Note that + // we don't expose these numbers to the user, so the only requirement is that + // we have some unique ID for each union case/possibility. The field tag + // numbers are already present and are easy to use so there's no reason to + // create a separate ID space. In addition, using the field tag number here + // lets us easily look up the field in the oneof accessor. + hd->oneof_case_num = upb_fielddef_number(f); + if (upb_fielddef_type(f) == UPB_TYPE_MESSAGE) { + hd->md = upb_fielddef_msgsubdef(f); + } else { + hd->md = NULL; + } + upb_handlers_addcleanup(h, hd, free); + return hd; +} + // A handler that starts a repeated field. Gets the Repeated*Field instance for // this field (such an instance always exists even in an empty message). static void *startseq_handler(void* closure, const void* hd) { @@ -67,8 +97,7 @@ static void *startseq_handler(void* closure, const void* hd) { return (void*)DEREF(msg, *ofs, VALUE); } -// Handlers that append primitive values to a repeated field (a regular Ruby -// array for now). +// Handlers that append primitive values to a repeated field. #define DEFINE_APPEND_HANDLER(type, ctype) \ static bool append##type##_handler(void *closure, const void *hd, \ ctype val) { \ @@ -85,7 +114,7 @@ DEFINE_APPEND_HANDLER(int64, int64_t) DEFINE_APPEND_HANDLER(uint64, uint64_t) DEFINE_APPEND_HANDLER(double, double) -// Appends a string to a repeated field (a regular Ruby array for now). +// Appends a string to a repeated field. static void* appendstr_handler(void *closure, const void *hd, size_t size_hint) { @@ -96,7 +125,7 @@ static void* appendstr_handler(void *closure, return (void*)str; } -// Appends a 'bytes' string to a repeated field (a regular Ruby array for now). +// Appends a 'bytes' string to a repeated field. static void* appendbytes_handler(void *closure, const void *hd, size_t size_hint) { @@ -179,7 +208,11 @@ typedef struct { size_t ofs; upb_fieldtype_t key_field_type; upb_fieldtype_t value_field_type; - VALUE value_field_typeclass; + + // We know that we can hold this reference because the handlerdata has the + // same lifetime as the upb_handlers struct, and the upb_handlers struct holds + // a reference to the upb_msgdef, which in turn has references to its subdefs. + const upb_def* value_field_subdef; } map_handlerdata_t; // Temporary frame for map parsing: at the beginning of a map entry message, a @@ -219,8 +252,15 @@ static bool endmap_handler(void *closure, const void *hd, upb_status* s) { VALUE key = native_slot_get( mapdata->key_field_type, Qnil, &frame->key_storage); + + VALUE value_field_typeclass = Qnil; + if (mapdata->value_field_type == UPB_TYPE_MESSAGE || + mapdata->value_field_type == UPB_TYPE_ENUM) { + value_field_typeclass = get_def_obj(mapdata->value_field_subdef); + } + VALUE value = native_slot_get( - mapdata->value_field_type, mapdata->value_field_typeclass, + mapdata->value_field_type, value_field_typeclass, &frame->value_storage); Map_index_set(frame->map, key, value); @@ -251,21 +291,90 @@ static map_handlerdata_t* new_map_handlerdata( MAP_VALUE_FIELD); assert(value_field != NULL); hd->value_field_type = upb_fielddef_type(value_field); - hd->value_field_typeclass = field_type_class(value_field); - - // Ensure that value_field_typeclass is properly GC-rooted. We must do this - // because we hold a reference to the Ruby class in the handlerdata, which is - // owned by the handlers. The handlers are owned by *this* message's Ruby - // object, but each Ruby object is rooted independently at the def -> Ruby - // object map. So we have to ensure that the Ruby objects we depend on will - // stick around as long as we're around. - if (hd->value_field_typeclass != Qnil) { - rb_ary_push(desc->typeclass_references, hd->value_field_typeclass); - } + hd->value_field_subdef = upb_fielddef_subdef(value_field); return hd; } +// Handlers that set primitive values in oneofs. +#define DEFINE_ONEOF_HANDLER(type, ctype) \ + static bool oneof##type##_handler(void *closure, const void *hd, \ + ctype val) { \ + const oneof_handlerdata_t *oneofdata = hd; \ + DEREF(closure, oneofdata->case_ofs, uint32_t) = \ + oneofdata->oneof_case_num; \ + DEREF(closure, oneofdata->ofs, ctype) = val; \ + return true; \ + } + +DEFINE_ONEOF_HANDLER(bool, bool) +DEFINE_ONEOF_HANDLER(int32, int32_t) +DEFINE_ONEOF_HANDLER(uint32, uint32_t) +DEFINE_ONEOF_HANDLER(float, float) +DEFINE_ONEOF_HANDLER(int64, int64_t) +DEFINE_ONEOF_HANDLER(uint64, uint64_t) +DEFINE_ONEOF_HANDLER(double, double) + +#undef DEFINE_ONEOF_HANDLER + +// Handlers for strings in a oneof. +static void *oneofstr_handler(void *closure, + const void *hd, + size_t size_hint) { + MessageHeader* msg = closure; + const oneof_handlerdata_t *oneofdata = hd; + VALUE str = rb_str_new2(""); + rb_enc_associate(str, kRubyStringUtf8Encoding); + DEREF(msg, oneofdata->case_ofs, uint32_t) = + oneofdata->oneof_case_num; + DEREF(msg, oneofdata->ofs, VALUE) = str; + return (void*)str; +} + +static void *oneofbytes_handler(void *closure, + const void *hd, + size_t size_hint) { + MessageHeader* msg = closure; + const oneof_handlerdata_t *oneofdata = hd; + VALUE str = rb_str_new2(""); + rb_enc_associate(str, kRubyString8bitEncoding); + DEREF(msg, oneofdata->case_ofs, uint32_t) = + oneofdata->oneof_case_num; + DEREF(msg, oneofdata->ofs, VALUE) = str; + return (void*)str; +} + +// Handler for a submessage field in a oneof. +static void *oneofsubmsg_handler(void *closure, + const void *hd) { + MessageHeader* msg = closure; + const oneof_handlerdata_t *oneofdata = hd; + uint32_t oldcase = DEREF(msg, oneofdata->case_ofs, uint32_t); + + VALUE subdesc = + get_def_obj((void*)oneofdata->md); + VALUE subklass = Descriptor_msgclass(subdesc); + + if (oldcase != oneofdata->oneof_case_num || + DEREF(msg, oneofdata->ofs, VALUE) == Qnil) { + DEREF(msg, oneofdata->ofs, VALUE) = + rb_class_new_instance(0, NULL, subklass); + } + // Set the oneof case *after* allocating the new class instance -- otherwise, + // if the Ruby GC is invoked as part of a call into the VM, it might invoke + // our mark routines, and our mark routines might see the case value + // indicating a VALUE is present and expect a valid VALUE. See comment in + // layout_set() for more detail: basically, the change to the value and the + // case must be atomic w.r.t. the Ruby VM. + DEREF(msg, oneofdata->case_ofs, uint32_t) = + oneofdata->oneof_case_num; + + VALUE submsg_rb = DEREF(msg, oneofdata->ofs, VALUE); + MessageHeader* submsg; + TypedData_Get_Struct(submsg_rb, MessageHeader, &Message_type, submsg); + return submsg; +} + // Set up handlers for a repeated field. static void add_handlers_for_repeated_field(upb_handlers *h, const upb_fielddef *f, @@ -383,6 +492,53 @@ static void add_handlers_for_mapentry(const upb_msgdef* msgdef, offsetof(map_parse_frame_t, value_storage)); } +// Set up handlers for a oneof field. +static void add_handlers_for_oneof_field(upb_handlers *h, + const upb_fielddef *f, + size_t offset, + size_t oneof_case_offset) { + + upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER; + upb_handlerattr_sethandlerdata( + &attr, newoneofhandlerdata(h, offset, oneof_case_offset, f)); + + switch (upb_fielddef_type(f)) { + +#define SET_HANDLER(utype, ltype) \ + case utype: \ + upb_handlers_set##ltype(h, f, oneof##ltype##_handler, &attr); \ + break; + + SET_HANDLER(UPB_TYPE_BOOL, bool); + SET_HANDLER(UPB_TYPE_INT32, int32); + SET_HANDLER(UPB_TYPE_UINT32, uint32); + SET_HANDLER(UPB_TYPE_ENUM, int32); + SET_HANDLER(UPB_TYPE_FLOAT, float); + SET_HANDLER(UPB_TYPE_INT64, int64); + SET_HANDLER(UPB_TYPE_UINT64, uint64); + SET_HANDLER(UPB_TYPE_DOUBLE, double); + +#undef SET_HANDLER + + case UPB_TYPE_STRING: + case UPB_TYPE_BYTES: { + bool is_bytes = upb_fielddef_type(f) == UPB_TYPE_BYTES; + upb_handlers_setstartstr(h, f, is_bytes ? + oneofbytes_handler : oneofstr_handler, + &attr); + upb_handlers_setstring(h, f, stringdata_handler, NULL); + break; + } + case UPB_TYPE_MESSAGE: { + upb_handlers_setstartsubmsg(h, f, oneofsubmsg_handler, &attr); + break; + } + } + + upb_handlerattr_uninit(&attr); +} + + static void add_handlers_for_message(const void *closure, upb_handlers *h) { const upb_msgdef* msgdef = upb_handlers_msgdef(h); Descriptor* desc = ruby_to_Descriptor(get_def_obj((void*)msgdef)); @@ -402,16 +558,20 @@ static void add_handlers_for_message(const void *closure, upb_handlers *h) { desc->layout = create_layout(desc->msgdef); } - upb_msg_iter i; - - for (upb_msg_begin(&i, desc->msgdef); - !upb_msg_done(&i); - upb_msg_next(&i)) { + upb_msg_field_iter i; + for (upb_msg_field_begin(&i, desc->msgdef); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { const upb_fielddef *f = upb_msg_iter_field(&i); - size_t offset = desc->layout->offsets[upb_fielddef_index(f)] + + size_t offset = desc->layout->fields[upb_fielddef_index(f)].offset + sizeof(MessageHeader); - if (is_map_field(f)) { + if (upb_fielddef_containingoneof(f)) { + size_t oneof_case_offset = + desc->layout->fields[upb_fielddef_index(f)].case_offset + + sizeof(MessageHeader); + add_handlers_for_oneof_field(h, f, offset, oneof_case_offset); + } else if (is_map_field(f)) { add_handlers_for_mapfield(h, f, offset, desc); } else if (upb_fielddef_isseq(f)) { add_handlers_for_repeated_field(h, f, offset); @@ -804,13 +964,28 @@ static void putmsg(VALUE msg_rb, const Descriptor* desc, MessageHeader* msg; TypedData_Get_Struct(msg_rb, MessageHeader, &Message_type, msg); - upb_msg_iter i; - for (upb_msg_begin(&i, desc->msgdef); - !upb_msg_done(&i); - upb_msg_next(&i)) { + upb_msg_field_iter i; + for (upb_msg_field_begin(&i, desc->msgdef); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { upb_fielddef *f = upb_msg_iter_field(&i); uint32_t offset = - desc->layout->offsets[upb_fielddef_index(f)] + sizeof(MessageHeader); + desc->layout->fields[upb_fielddef_index(f)].offset + + sizeof(MessageHeader); + + if (upb_fielddef_containingoneof(f)) { + uint32_t oneof_case_offset = + desc->layout->fields[upb_fielddef_index(f)].case_offset + + sizeof(MessageHeader); + // For a oneof, check that this field is actually present -- skip all the + // below if not. + if (DEREF(msg, oneof_case_offset, uint32_t) != + upb_fielddef_number(f)) { + continue; + } + // Otherwise, fall through to the appropriate singular-field handler + // below. + } if (is_map_field(f)) { VALUE map = DEREF(msg, offset, VALUE); diff --git a/ruby/ext/google/protobuf_c/map.c b/ruby/ext/google/protobuf_c/map.c index 4ee71d18..12e7a9d9 100644 --- a/ruby/ext/google/protobuf_c/map.c +++ b/ruby/ext/google/protobuf_c/map.c @@ -683,7 +683,7 @@ VALUE Map_inspect(VALUE _self) { first = false; } str = rb_str_append(str, rb_funcall(key, inspect_sym, 0)); - str = rb_str_cat2(str, " => "); + str = rb_str_cat2(str, "=>"); str = rb_str_append(str, rb_funcall(value, inspect_sym, 0)); } diff --git a/ruby/ext/google/protobuf_c/message.c b/ruby/ext/google/protobuf_c/message.c index ee8881d4..7e58a617 100644 --- a/ruby/ext/google/protobuf_c/message.c +++ b/ruby/ext/google/protobuf_c/message.c @@ -70,6 +70,35 @@ VALUE Message_alloc(VALUE klass) { return ret; } +static VALUE which_oneof_field(MessageHeader* self, const upb_oneofdef* o) { + // If no fields in the oneof, always nil. + if (upb_oneofdef_numfields(o) == 0) { + return Qnil; + } + // Grab the first field in the oneof so we can get its layout info to find the + // oneof_case field. + upb_oneof_iter it; + upb_oneof_begin(&it, o); + assert(!upb_oneof_done(&it)); + const upb_fielddef* first_field = upb_oneof_iter_field(&it); + assert(upb_fielddef_containingoneof(first_field) != NULL); + + size_t case_ofs = + self->descriptor->layout-> + fields[upb_fielddef_index(first_field)].case_offset; + uint32_t oneof_case = *((uint32_t*)(Message_data(self) + case_ofs)); + + if (oneof_case == ONEOF_CASE_NONE) { + return Qnil; + } + + // oneof_case is a field index, so find that field. + const upb_fielddef* f = upb_oneofdef_itof(o, oneof_case); + assert(f != NULL); + + return ID2SYM(rb_intern(upb_fielddef_name(f))); +} + /* * call-seq: * Message.method_missing(*args) @@ -82,6 +111,10 @@ VALUE Message_alloc(VALUE klass) { * * msg.foo = 42 * puts msg.foo + * + * This method also provides read-only accessors for oneofs. If a oneof exists + * with name 'my_oneof', then msg.my_oneof will return a Ruby symbol equal to + * the name of the field in that oneof that is currently set, or nil if none. */ VALUE Message_method_missing(int argc, VALUE* argv, VALUE _self) { MessageHeader* self; @@ -104,6 +137,17 @@ VALUE Message_method_missing(int argc, VALUE* argv, VALUE _self) { name_len--; } + // Check for a oneof name first. + const upb_oneofdef* o = upb_msgdef_ntoo(self->descriptor->msgdef, + name, name_len); + if (o != NULL) { + if (setter) { + rb_raise(rb_eRuntimeError, "Oneof accessors are read-only."); + } + return which_oneof_field(self, o); + } + + // Otherwise, check for a field with that name. const upb_fielddef* f = upb_msgdef_ntof(self->descriptor->msgdef, name, name_len); diff --git a/ruby/ext/google/protobuf_c/protobuf.c b/ruby/ext/google/protobuf_c/protobuf.c index 30552705..d2d35033 100644 --- a/ruby/ext/google/protobuf_c/protobuf.c +++ b/ruby/ext/google/protobuf_c/protobuf.c @@ -77,8 +77,10 @@ void Init_protobuf_c() { DescriptorPool_register(protobuf); Descriptor_register(protobuf); FieldDescriptor_register(protobuf); + OneofDescriptor_register(protobuf); EnumDescriptor_register(protobuf); MessageBuilderContext_register(internal); + OneofBuilderContext_register(internal); EnumBuilderContext_register(internal); Builder_register(internal); RepeatedField_register(protobuf); diff --git a/ruby/ext/google/protobuf_c/protobuf.h b/ruby/ext/google/protobuf_c/protobuf.h index 88ae62e4..d8a327aa 100644 --- a/ruby/ext/google/protobuf_c/protobuf.h +++ b/ruby/ext/google/protobuf_c/protobuf.h @@ -43,6 +43,7 @@ struct Descriptor; struct FieldDescriptor; struct EnumDescriptor; struct MessageLayout; +struct MessageField; struct MessageHeader; struct MessageBuilderContext; struct EnumBuilderContext; @@ -51,10 +52,13 @@ struct Builder; typedef struct DescriptorPool DescriptorPool; typedef struct Descriptor Descriptor; typedef struct FieldDescriptor FieldDescriptor; +typedef struct OneofDescriptor OneofDescriptor; typedef struct EnumDescriptor EnumDescriptor; typedef struct MessageLayout MessageLayout; +typedef struct MessageField MessageField; typedef struct MessageHeader MessageHeader; typedef struct MessageBuilderContext MessageBuilderContext; +typedef struct OneofBuilderContext OneofBuilderContext; typedef struct EnumBuilderContext EnumBuilderContext; typedef struct Builder Builder; @@ -120,6 +124,10 @@ struct FieldDescriptor { const upb_fielddef* fielddef; }; +struct OneofDescriptor { + const upb_oneofdef* oneofdef; +}; + struct EnumDescriptor { const upb_enumdef* enumdef; VALUE module; // begins as nil @@ -130,6 +138,11 @@ struct MessageBuilderContext { VALUE builder; }; +struct OneofBuilderContext { + VALUE descriptor; + VALUE builder; +}; + struct EnumBuilderContext { VALUE enumdesc; }; @@ -144,6 +157,7 @@ extern VALUE cDescriptor; extern VALUE cFieldDescriptor; extern VALUE cEnumDescriptor; extern VALUE cMessageBuilderContext; +extern VALUE cOneofBuilderContext; extern VALUE cEnumBuilderContext; extern VALUE cBuilder; @@ -175,6 +189,9 @@ VALUE Descriptor_name_set(VALUE _self, VALUE str); VALUE Descriptor_each(VALUE _self); VALUE Descriptor_lookup(VALUE _self, VALUE name); VALUE Descriptor_add_field(VALUE _self, VALUE obj); +VALUE Descriptor_add_oneof(VALUE _self, VALUE obj); +VALUE Descriptor_each_oneof(VALUE _self); +VALUE Descriptor_lookup_oneof(VALUE _self, VALUE name); VALUE Descriptor_msgclass(VALUE _self); extern const rb_data_type_t _Descriptor_type; @@ -199,6 +216,16 @@ VALUE FieldDescriptor_set(VALUE _self, VALUE msg_rb, VALUE value); upb_fieldtype_t ruby_to_fieldtype(VALUE type); VALUE fieldtype_to_ruby(upb_fieldtype_t type); +void OneofDescriptor_mark(void* _self); +void OneofDescriptor_free(void* _self); +VALUE OneofDescriptor_alloc(VALUE klass); +void OneofDescriptor_register(VALUE module); +OneofDescriptor* ruby_to_OneofDescriptor(VALUE value); +VALUE OneofDescriptor_name(VALUE _self); +VALUE OneofDescriptor_name_set(VALUE _self, VALUE value); +VALUE OneofDescriptor_add_field(VALUE _self, VALUE field); +VALUE OneofDescriptor_each(VALUE _self, VALUE field); + void EnumDescriptor_mark(void* _self); void EnumDescriptor_free(void* _self); VALUE EnumDescriptor_alloc(VALUE klass); @@ -225,6 +252,17 @@ VALUE MessageBuilderContext_optional(int argc, VALUE* argv, VALUE _self); VALUE MessageBuilderContext_required(int argc, VALUE* argv, VALUE _self); VALUE MessageBuilderContext_repeated(int argc, VALUE* argv, VALUE _self); VALUE MessageBuilderContext_map(int argc, VALUE* argv, VALUE _self); +VALUE MessageBuilderContext_oneof(VALUE _self, VALUE name); + +void OneofBuilderContext_mark(void* _self); +void OneofBuilderContext_free(void* _self); +VALUE OneofBuilderContext_alloc(VALUE klass); +void OneofBuilderContext_register(VALUE module); +OneofBuilderContext* ruby_to_OneofBuilderContext(VALUE value); +VALUE OneofBuilderContext_initialize(VALUE _self, + VALUE descriptor, + VALUE builder); +VALUE OneofBuilderContext_optional(int argc, VALUE* argv, VALUE _self); void EnumBuilderContext_mark(void* _self); void EnumBuilderContext_free(void* _self); @@ -247,13 +285,22 @@ VALUE Builder_finalize_to_pool(VALUE _self, VALUE pool_rb); // Native slot storage abstraction. // ----------------------------------------------------------------------------- -#define NATIVE_SLOT_MAX_SIZE sizeof(void*) +#define NATIVE_SLOT_MAX_SIZE sizeof(uint64_t) size_t native_slot_size(upb_fieldtype_t type); void native_slot_set(upb_fieldtype_t type, VALUE type_class, void* memory, VALUE value); +// Atomically (with respect to Ruby VM calls) either update the value and set a +// oneof case, or do neither. If |case_memory| is null, then no case value is +// set. +void native_slot_set_value_and_case(upb_fieldtype_t type, + VALUE type_class, + void* memory, + VALUE value, + uint32_t* case_memory, + uint32_t case_number); VALUE native_slot_get(upb_fieldtype_t type, VALUE type_class, const void* memory); @@ -275,6 +322,11 @@ VALUE field_type_class(const upb_fielddef* field); #define MAP_KEY_FIELD 1 #define MAP_VALUE_FIELD 2 +// Oneof case slot value to indicate that no oneof case is set. The value `0` is +// safe because field numbers are used as case identifiers, and no field can +// have a number of 0. +#define ONEOF_CASE_NONE 0 + // These operate on a map field (i.e., a repeated field of submessages whose // submessage type is a map-entry msgdef). bool is_map_field(const upb_fielddef* field); @@ -384,9 +436,16 @@ VALUE Map_iter_value(Map_iter* iter); // Message layout / storage. // ----------------------------------------------------------------------------- +#define MESSAGE_FIELD_NO_CASE ((size_t)-1) + +struct MessageField { + size_t offset; + size_t case_offset; // for oneofs, a uint32. Else, MESSAGE_FIELD_NO_CASE. +}; + struct MessageLayout { const upb_msgdef* msgdef; - size_t* offsets; + MessageField* fields; size_t size; }; diff --git a/ruby/ext/google/protobuf_c/repeated_field.c b/ruby/ext/google/protobuf_c/repeated_field.c index 6e3f0bc7..8cf2e29b 100644 --- a/ruby/ext/google/protobuf_c/repeated_field.c +++ b/ruby/ext/google/protobuf_c/repeated_field.c @@ -318,6 +318,29 @@ VALUE RepeatedField_deep_copy(VALUE _self) { /* * call-seq: + * RepeatedField.to_ary => array + * + * Used when converted implicitly into array, e.g. compared to an Array. + * Also called as a fallback of Object#to_a + */ +VALUE RepeatedField_to_ary(VALUE _self) { + RepeatedField* self = ruby_to_RepeatedField(_self); + upb_fieldtype_t field_type = self->field_type; + + size_t elem_size = native_slot_size(field_type); + size_t off = 0; + VALUE ary = rb_ary_new2(self->size); + for (int i = 0; i < self->size; i++, off += elem_size) { + void* mem = ((uint8_t *)self->elements) + off; + VALUE elem = native_slot_get(field_type, self->field_type_class, mem); + + rb_ary_push(ary, elem); + } + return ary; +} + +/* + * call-seq: * RepeatedField.==(other) => boolean * * Compares this repeated field to another. Repeated fields are equal if their @@ -335,15 +358,9 @@ VALUE RepeatedField_eq(VALUE _self, VALUE _other) { } RepeatedField* self = ruby_to_RepeatedField(_self); - // Inefficient but workable: to support comparison to a generic array, we - // build a temporary RepeatedField of our type. if (TYPE(_other) == T_ARRAY) { - VALUE new_rptfield = RepeatedField_new_this_type(_self); - for (int i = 0; i < RARRAY_LEN(_other); i++) { - VALUE elem = rb_ary_entry(_other, i); - RepeatedField_push(new_rptfield, elem); - } - _other = new_rptfield; + VALUE self_ary = RepeatedField_to_ary(_self); + return rb_equal(self_ary, _other); } RepeatedField* other = ruby_to_RepeatedField(_other); @@ -401,29 +418,8 @@ VALUE RepeatedField_hash(VALUE _self) { * representation computed by its own #inspect method. */ VALUE RepeatedField_inspect(VALUE _self) { - RepeatedField* self = ruby_to_RepeatedField(_self); - - VALUE str = rb_str_new2("["); - - bool first = true; - - upb_fieldtype_t field_type = self->field_type; - VALUE field_type_class = self->field_type_class; - size_t elem_size = native_slot_size(field_type); - size_t off = 0; - for (int i = 0; i < self->size; i++, off += elem_size) { - void* mem = ((uint8_t *)self->elements) + off; - VALUE elem = native_slot_get(field_type, field_type_class, mem); - if (!first) { - str = rb_str_cat2(str, ", "); - } else { - first = false; - } - str = rb_str_append(str, rb_funcall(elem, rb_intern("inspect"), 0)); - } - - str = rb_str_cat2(str, "]"); - return str; + VALUE self_ary = RepeatedField_to_ary(_self); + return rb_funcall(self_ary, rb_intern("inspect"), 0); } /* @@ -594,6 +590,7 @@ void RepeatedField_register(VALUE module) { // Also define #clone so that we don't inherit Object#clone. rb_define_method(klass, "clone", RepeatedField_dup, 0); rb_define_method(klass, "==", RepeatedField_eq, 1); + rb_define_method(klass, "to_ary", RepeatedField_to_ary, 0); rb_define_method(klass, "hash", RepeatedField_hash, 0); rb_define_method(klass, "inspect", RepeatedField_inspect, 0); rb_define_method(klass, "+", RepeatedField_plus, 1); diff --git a/ruby/ext/google/protobuf_c/storage.c b/ruby/ext/google/protobuf_c/storage.c index 14f49d44..5b1549d2 100644 --- a/ruby/ext/google/protobuf_c/storage.c +++ b/ruby/ext/google/protobuf_c/storage.c @@ -109,6 +109,17 @@ void native_slot_validate_string_encoding(upb_fieldtype_t type, VALUE value) { void native_slot_set(upb_fieldtype_t type, VALUE type_class, void* memory, VALUE value) { + native_slot_set_value_and_case(type, type_class, memory, value, NULL, 0); +} + +void native_slot_set_value_and_case(upb_fieldtype_t type, VALUE type_class, + void* memory, VALUE value, + uint32_t* case_memory, + uint32_t case_number) { + // Note that in order to atomically change the value in memory and the case + // value (w.r.t. Ruby VM calls), we must set the value at |memory| only after + // all Ruby VM calls are complete. The case is then set at the bottom of this + // function. switch (type) { case UPB_TYPE_FLOAT: if (!is_ruby_num(value)) { @@ -198,6 +209,10 @@ void native_slot_set(upb_fieldtype_t type, VALUE type_class, default: break; } + + if (case_memory != NULL) { + *case_memory = case_number; + } } VALUE native_slot_get(upb_fieldtype_t type, @@ -366,24 +381,94 @@ const upb_fielddef* map_entry_value(const upb_msgdef* msgdef) { // Memory layout management. // ----------------------------------------------------------------------------- +static size_t align_up_to(size_t offset, size_t granularity) { + // Granularity must be a power of two. + return (offset + granularity - 1) & ~(granularity - 1); +} + MessageLayout* create_layout(const upb_msgdef* msgdef) { MessageLayout* layout = ALLOC(MessageLayout); int nfields = upb_msgdef_numfields(msgdef); - layout->offsets = ALLOC_N(size_t, nfields); + layout->fields = ALLOC_N(MessageField, nfields); - upb_msg_iter it; + upb_msg_field_iter it; size_t off = 0; - for (upb_msg_begin(&it, msgdef); !upb_msg_done(&it); upb_msg_next(&it)) { + for (upb_msg_field_begin(&it, msgdef); + !upb_msg_field_done(&it); + upb_msg_field_next(&it)) { const upb_fielddef* field = upb_msg_iter_field(&it); + + if (upb_fielddef_containingoneof(field)) { + // Oneofs are handled separately below. + continue; + } + + // Allocate |field_size| bytes for this field in the layout. size_t field_size = 0; if (upb_fielddef_label(field) == UPB_LABEL_REPEATED) { field_size = sizeof(VALUE); } else { field_size = native_slot_size(upb_fielddef_type(field)); } - // align current offset + // Align current offset up to |size| granularity. + off = align_up_to(off, field_size); + layout->fields[upb_fielddef_index(field)].offset = off; + layout->fields[upb_fielddef_index(field)].case_offset = MESSAGE_FIELD_NO_CASE; + off += field_size; + } + + // Handle oneofs now -- we iterate over oneofs specifically and allocate only + // one slot per oneof. + // + // We assign all value slots first, then pack the 'case' fields at the end, + // since in the common case (modern 64-bit platform) these are 8 bytes and 4 + // bytes respectively and we want to avoid alignment overhead. + // + // Note that we reserve 4 bytes (a uint32) per 'case' slot because the value + // space for oneof cases is conceptually as wide as field tag numbers. In + // practice, it's unlikely that a oneof would have more than e.g. 256 or 64K + // members (8 or 16 bits respectively), so conceivably we could assign + // consecutive case numbers and then pick a smaller oneof case slot size, but + // the complexity to implement this indirection is probably not worthwhile. + upb_msg_oneof_iter oit; + for (upb_msg_oneof_begin(&oit, msgdef); + !upb_msg_oneof_done(&oit); + upb_msg_oneof_next(&oit)) { + const upb_oneofdef* oneof = upb_msg_iter_oneof(&oit); + + // Always allocate NATIVE_SLOT_MAX_SIZE bytes, but share the slot between + // all fields. + size_t field_size = NATIVE_SLOT_MAX_SIZE; + // Align the offset. + off = align_up_to(off, field_size); + // Assign all fields in the oneof this same offset. + upb_oneof_iter fit; + for (upb_oneof_begin(&fit, oneof); + !upb_oneof_done(&fit); + upb_oneof_next(&fit)) { + const upb_fielddef* field = upb_oneof_iter_field(&fit); + layout->fields[upb_fielddef_index(field)].offset = off; + } + off += field_size; + } + + // Now the case fields. + for (upb_msg_oneof_begin(&oit, msgdef); + !upb_msg_oneof_done(&oit); + upb_msg_oneof_next(&oit)) { + const upb_oneofdef* oneof = upb_msg_iter_oneof(&oit); + + size_t field_size = sizeof(uint32_t); + // Align the offset. off = (off + field_size - 1) & ~(field_size - 1); - layout->offsets[upb_fielddef_index(field)] = off; + // Assign all fields in the oneof this same offset. + upb_oneof_iter fit; + for (upb_oneof_begin(&fit, oneof); + !upb_oneof_done(&fit); + upb_oneof_next(&fit)) { + const upb_fielddef* field = upb_oneof_iter_field(&fit); + layout->fields[upb_fielddef_index(field)].case_offset = off; + } off += field_size; } @@ -396,7 +481,7 @@ MessageLayout* create_layout(const upb_msgdef* msgdef) { } void free_layout(MessageLayout* layout) { - xfree(layout->offsets); + xfree(layout->fields); upb_msgdef_unref(layout->msgdef, &layout->msgdef); xfree(layout); } @@ -415,12 +500,35 @@ VALUE field_type_class(const upb_fielddef* field) { return type_class; } +static void* slot_memory(MessageLayout* layout, + const void* storage, + const upb_fielddef* field) { + return ((uint8_t *)storage) + + layout->fields[upb_fielddef_index(field)].offset; +} + +static uint32_t* slot_oneof_case(MessageLayout* layout, + const void* storage, + const upb_fielddef* field) { + return (uint32_t *)(((uint8_t *)storage) + + layout->fields[upb_fielddef_index(field)].case_offset); +} + + VALUE layout_get(MessageLayout* layout, const void* storage, const upb_fielddef* field) { - void* memory = ((uint8_t *)storage) + - layout->offsets[upb_fielddef_index(field)]; - if (upb_fielddef_label(field) == UPB_LABEL_REPEATED) { + void* memory = slot_memory(layout, storage, field); + uint32_t* oneof_case = slot_oneof_case(layout, storage, field); + + if (upb_fielddef_containingoneof(field)) { + if (*oneof_case != upb_fielddef_number(field)) { + return Qnil; + } + return native_slot_get(upb_fielddef_type(field), + field_type_class(field), + memory); + } else if (upb_fielddef_label(field) == UPB_LABEL_REPEATED) { return *((VALUE *)memory); } else { return native_slot_get(upb_fielddef_type(field), @@ -484,9 +592,33 @@ void layout_set(MessageLayout* layout, void* storage, const upb_fielddef* field, VALUE val) { - void* memory = ((uint8_t *)storage) + - layout->offsets[upb_fielddef_index(field)]; - if (is_map_field(field)) { + void* memory = slot_memory(layout, storage, field); + uint32_t* oneof_case = slot_oneof_case(layout, storage, field); + + if (upb_fielddef_containingoneof(field)) { + if (val == Qnil) { + // Assigning nil to a oneof field clears the oneof completely. + *oneof_case = ONEOF_CASE_NONE; + memset(memory, 0, NATIVE_SLOT_MAX_SIZE); + } else { + // The transition between field types for a single oneof (union) slot is + // somewhat complex because we need to ensure that a GC triggered at any + // point by a call into the Ruby VM sees a valid state for this field and + // does not either go off into the weeds (following what it thinks is a + // VALUE but is actually a different field type) or miss an object (seeing + // what it thinks is a primitive field but is actually a VALUE for the new + // field type). + // + // In order for the transition to be safe, the oneof case slot must be in + // sync with the value slot whenever the Ruby VM has been called. Thus, we + // use native_slot_set_value_and_case(), which ensures that both the value + // and case number are altered atomically (w.r.t. the Ruby VM). + native_slot_set_value_and_case( + upb_fielddef_type(field), field_type_class(field), + memory, val, + oneof_case, upb_fielddef_number(field)); + } + } else if (is_map_field(field)) { check_map_field_type(val, field); DEREF(memory, VALUE) = val; } else if (upb_fielddef_label(field) == UPB_LABEL_REPEATED) { @@ -500,15 +632,18 @@ void layout_set(MessageLayout* layout, void layout_init(MessageLayout* layout, void* storage) { - upb_msg_iter it; - for (upb_msg_begin(&it, layout->msgdef); - !upb_msg_done(&it); - upb_msg_next(&it)) { + upb_msg_field_iter it; + for (upb_msg_field_begin(&it, layout->msgdef); + !upb_msg_field_done(&it); + upb_msg_field_next(&it)) { const upb_fielddef* field = upb_msg_iter_field(&it); - void* memory = ((uint8_t *)storage) + - layout->offsets[upb_fielddef_index(field)]; + void* memory = slot_memory(layout, storage, field); + uint32_t* oneof_case = slot_oneof_case(layout, storage, field); - if (is_map_field(field)) { + if (upb_fielddef_containingoneof(field)) { + memset(memory, 0, NATIVE_SLOT_MAX_SIZE); + *oneof_case = ONEOF_CASE_NONE; + } else if (is_map_field(field)) { VALUE map = Qnil; const upb_fielddef* key_field = map_field_key(field); @@ -555,15 +690,19 @@ void layout_init(MessageLayout* layout, } void layout_mark(MessageLayout* layout, void* storage) { - upb_msg_iter it; - for (upb_msg_begin(&it, layout->msgdef); - !upb_msg_done(&it); - upb_msg_next(&it)) { + upb_msg_field_iter it; + for (upb_msg_field_begin(&it, layout->msgdef); + !upb_msg_field_done(&it); + upb_msg_field_next(&it)) { const upb_fielddef* field = upb_msg_iter_field(&it); - void* memory = ((uint8_t *)storage) + - layout->offsets[upb_fielddef_index(field)]; + void* memory = slot_memory(layout, storage, field); + uint32_t* oneof_case = slot_oneof_case(layout, storage, field); - if (upb_fielddef_label(field) == UPB_LABEL_REPEATED) { + if (upb_fielddef_containingoneof(field)) { + if (*oneof_case == upb_fielddef_number(field)) { + native_slot_mark(upb_fielddef_type(field), memory); + } + } else if (upb_fielddef_label(field) == UPB_LABEL_REPEATED) { rb_gc_mark(DEREF(memory, VALUE)); } else { native_slot_mark(upb_fielddef_type(field), memory); @@ -572,17 +711,23 @@ void layout_mark(MessageLayout* layout, void* storage) { } void layout_dup(MessageLayout* layout, void* to, void* from) { - upb_msg_iter it; - for (upb_msg_begin(&it, layout->msgdef); - !upb_msg_done(&it); - upb_msg_next(&it)) { + upb_msg_field_iter it; + for (upb_msg_field_begin(&it, layout->msgdef); + !upb_msg_field_done(&it); + upb_msg_field_next(&it)) { const upb_fielddef* field = upb_msg_iter_field(&it); - void* to_memory = ((uint8_t *)to) + - layout->offsets[upb_fielddef_index(field)]; - void* from_memory = ((uint8_t *)from) + - layout->offsets[upb_fielddef_index(field)]; - if (is_map_field(field)) { + void* to_memory = slot_memory(layout, to, field); + uint32_t* to_oneof_case = slot_oneof_case(layout, to, field); + void* from_memory = slot_memory(layout, from, field); + uint32_t* from_oneof_case = slot_oneof_case(layout, from, field); + + if (upb_fielddef_containingoneof(field)) { + if (*from_oneof_case == upb_fielddef_number(field)) { + *to_oneof_case = *from_oneof_case; + native_slot_dup(upb_fielddef_type(field), to_memory, from_memory); + } + } else if (is_map_field(field)) { DEREF(to_memory, VALUE) = Map_dup(DEREF(from_memory, VALUE)); } else if (upb_fielddef_label(field) == UPB_LABEL_REPEATED) { DEREF(to_memory, VALUE) = RepeatedField_dup(DEREF(from_memory, VALUE)); @@ -593,17 +738,23 @@ void layout_dup(MessageLayout* layout, void* to, void* from) { } void layout_deep_copy(MessageLayout* layout, void* to, void* from) { - upb_msg_iter it; - for (upb_msg_begin(&it, layout->msgdef); - !upb_msg_done(&it); - upb_msg_next(&it)) { + upb_msg_field_iter it; + for (upb_msg_field_begin(&it, layout->msgdef); + !upb_msg_field_done(&it); + upb_msg_field_next(&it)) { const upb_fielddef* field = upb_msg_iter_field(&it); - void* to_memory = ((uint8_t *)to) + - layout->offsets[upb_fielddef_index(field)]; - void* from_memory = ((uint8_t *)from) + - layout->offsets[upb_fielddef_index(field)]; - if (is_map_field(field)) { + void* to_memory = slot_memory(layout, to, field); + uint32_t* to_oneof_case = slot_oneof_case(layout, to, field); + void* from_memory = slot_memory(layout, from, field); + uint32_t* from_oneof_case = slot_oneof_case(layout, from, field); + + if (upb_fielddef_containingoneof(field)) { + if (*from_oneof_case == upb_fielddef_number(field)) { + *to_oneof_case = *from_oneof_case; + native_slot_deep_copy(upb_fielddef_type(field), to_memory, from_memory); + } + } else if (is_map_field(field)) { DEREF(to_memory, VALUE) = Map_deep_copy(DEREF(from_memory, VALUE)); } else if (upb_fielddef_label(field) == UPB_LABEL_REPEATED) { @@ -616,22 +767,35 @@ void layout_deep_copy(MessageLayout* layout, void* to, void* from) { } VALUE layout_eq(MessageLayout* layout, void* msg1, void* msg2) { - upb_msg_iter it; - for (upb_msg_begin(&it, layout->msgdef); - !upb_msg_done(&it); - upb_msg_next(&it)) { + upb_msg_field_iter it; + for (upb_msg_field_begin(&it, layout->msgdef); + !upb_msg_field_done(&it); + upb_msg_field_next(&it)) { const upb_fielddef* field = upb_msg_iter_field(&it); - void* msg1_memory = ((uint8_t *)msg1) + - layout->offsets[upb_fielddef_index(field)]; - void* msg2_memory = ((uint8_t *)msg2) + - layout->offsets[upb_fielddef_index(field)]; - - if (is_map_field(field)) { - return Map_eq(DEREF(msg1_memory, VALUE), - DEREF(msg2_memory, VALUE)); + + void* msg1_memory = slot_memory(layout, msg1, field); + uint32_t* msg1_oneof_case = slot_oneof_case(layout, msg1, field); + void* msg2_memory = slot_memory(layout, msg2, field); + uint32_t* msg2_oneof_case = slot_oneof_case(layout, msg2, field); + + if (upb_fielddef_containingoneof(field)) { + if (*msg1_oneof_case != *msg2_oneof_case || + (*msg1_oneof_case == upb_fielddef_number(field) && + !native_slot_eq(upb_fielddef_type(field), + msg1_memory, + msg2_memory))) { + return Qfalse; + } + } else if (is_map_field(field)) { + if (!Map_eq(DEREF(msg1_memory, VALUE), + DEREF(msg2_memory, VALUE))) { + return Qfalse; + } } else if (upb_fielddef_label(field) == UPB_LABEL_REPEATED) { - return RepeatedField_eq(DEREF(msg1_memory, VALUE), - DEREF(msg2_memory, VALUE)); + if (!RepeatedField_eq(DEREF(msg1_memory, VALUE), + DEREF(msg2_memory, VALUE))) { + return Qfalse; + } } else { if (!native_slot_eq(upb_fielddef_type(field), msg1_memory, msg2_memory)) { @@ -643,12 +807,12 @@ VALUE layout_eq(MessageLayout* layout, void* msg1, void* msg2) { } VALUE layout_hash(MessageLayout* layout, void* storage) { - upb_msg_iter it; + upb_msg_field_iter it; st_index_t h = rb_hash_start(0); VALUE hash_sym = rb_intern("hash"); - for (upb_msg_begin(&it, layout->msgdef); - !upb_msg_done(&it); - upb_msg_next(&it)) { + for (upb_msg_field_begin(&it, layout->msgdef); + !upb_msg_field_done(&it); + upb_msg_field_next(&it)) { const upb_fielddef* field = upb_msg_iter_field(&it); VALUE field_val = layout_get(layout, storage, field); h = rb_hash_uint(h, NUM2LONG(rb_funcall(field_val, hash_sym, 0))); @@ -661,11 +825,11 @@ VALUE layout_hash(MessageLayout* layout, void* storage) { VALUE layout_inspect(MessageLayout* layout, void* storage) { VALUE str = rb_str_new2(""); - upb_msg_iter it; + upb_msg_field_iter it; bool first = true; - for (upb_msg_begin(&it, layout->msgdef); - !upb_msg_done(&it); - upb_msg_next(&it)) { + for (upb_msg_field_begin(&it, layout->msgdef); + !upb_msg_field_done(&it); + upb_msg_field_next(&it)) { const upb_fielddef* field = upb_msg_iter_field(&it); VALUE field_val = layout_get(layout, storage, field); diff --git a/ruby/ext/google/protobuf_c/upb.c b/ruby/ext/google/protobuf_c/upb.c index 571c809f..20bd76bc 100644 --- a/ruby/ext/google/protobuf_c/upb.c +++ b/ruby/ext/google/protobuf_c/upb.c @@ -210,6 +210,21 @@ static bool upb_validate_field(upb_fielddef *f, upb_status *s) { upb_fielddef_setdefaultint32(f, upb_fielddef_defaultint32(f)); } + // Ensure that MapEntry submessages only appear as repeated fields, not + // optional/required (singular) fields. + if (upb_fielddef_type(f) == UPB_TYPE_MESSAGE && + upb_fielddef_msgsubdef(f) != NULL) { + const upb_msgdef *subdef = upb_fielddef_msgsubdef(f); + if (upb_msgdef_mapentry(subdef) && !upb_fielddef_isseq(f)) { + upb_status_seterrf(s, + "Field %s refers to mapentry message but is not " + "a repeated field", + upb_fielddef_name(f) ? upb_fielddef_name(f) : + "(unnamed)"); + return false; + } + } + return true; } @@ -247,10 +262,12 @@ static bool assign_msg_indices(upb_msgdef *m, upb_status *s) { upb_fielddef **fields = malloc(n * sizeof(*fields)); if (!fields) return false; - upb_msg_iter j; + upb_msg_field_iter j; int i; m->submsg_field_count = 0; - for(i = 0, upb_msg_begin(&j, m); !upb_msg_done(&j); upb_msg_next(&j), i++) { + for(i = 0, upb_msg_field_begin(&j, m); + !upb_msg_field_done(&j); + upb_msg_field_next(&j), i++) { upb_fielddef *f = upb_msg_iter_field(&j); assert(f->msg.def == m); if (!upb_validate_field(f, s)) { @@ -286,7 +303,9 @@ static bool assign_msg_indices(upb_msgdef *m, upb_status *s) { upb_selector_t sel; upb_inttable_insert(&t, UPB_STARTMSG_SELECTOR, v); upb_inttable_insert(&t, UPB_ENDMSG_SELECTOR, v); - for(upb_msg_begin(&j, m); !upb_msg_done(&j); upb_msg_next(&j)) { + for(upb_msg_field_begin(&j, m); + !upb_msg_field_done(&j); + upb_msg_field_next(&j)) { upb_fielddef *f = upb_msg_iter_field(&j); // These calls will assert-fail in upb_table if the value already exists. TRY(UPB_HANDLER_INT32); @@ -544,6 +563,9 @@ static void visitfield(const upb_refcounted *r, upb_refcounted_visit *visit, if (upb_fielddef_containingtype(f)) { visit(r, UPB_UPCAST2(upb_fielddef_containingtype(f)), closure); } + if (upb_fielddef_containingoneof(f)) { + visit(r, UPB_UPCAST2(upb_fielddef_containingoneof(f)), closure); + } if (upb_fielddef_subdef(f)) { visit(r, UPB_UPCAST(upb_fielddef_subdef(f)), closure); } @@ -619,6 +641,7 @@ upb_fielddef *upb_fielddef_new(const void *owner) { } f->msg.def = NULL; f->sub.def = NULL; + f->oneof = NULL; f->subdef_is_symbolic = false; f->msg_is_symbolic = false; f->label_ = UPB_LABEL_OPTIONAL; @@ -748,6 +771,10 @@ const upb_msgdef *upb_fielddef_containingtype(const upb_fielddef *f) { return f->msg_is_symbolic ? NULL : f->msg.def; } +const upb_oneofdef *upb_fielddef_containingoneof(const upb_fielddef *f) { + return f->oneof; +} + upb_msgdef *upb_fielddef_containingtype_mutable(upb_fielddef *f) { return (upb_msgdef*)upb_fielddef_containingtype(f); } @@ -776,6 +803,10 @@ bool upb_fielddef_setcontainingtypename(upb_fielddef *f, const char *name, } bool upb_fielddef_setname(upb_fielddef *f, const char *name, upb_status *s) { + if (upb_fielddef_containingtype(f) || upb_fielddef_containingoneof(f)) { + upb_status_seterrmsg(s, "Already added to message or oneof"); + return false; + } return upb_def_setfullname(UPB_UPCAST(f), name, s); } @@ -1226,6 +1257,11 @@ bool upb_fielddef_isprimitive(const upb_fielddef *f) { return !upb_fielddef_isstring(f) && !upb_fielddef_issubmsg(f); } +bool upb_fielddef_ismap(const upb_fielddef *f) { + return upb_fielddef_isseq(f) && upb_fielddef_issubmsg(f) && + upb_msgdef_mapentry(upb_fielddef_msgsubdef(f)); +} + bool upb_fielddef_hassubdef(const upb_fielddef *f) { return upb_fielddef_issubmsg(f) || upb_fielddef_type(f) == UPB_TYPE_ENUM; } @@ -1247,15 +1283,25 @@ bool upb_fielddef_checkdescriptortype(int32_t type) { static void visitmsg(const upb_refcounted *r, upb_refcounted_visit *visit, void *closure) { const upb_msgdef *m = (const upb_msgdef*)r; - upb_msg_iter i; - for(upb_msg_begin(&i, m); !upb_msg_done(&i); upb_msg_next(&i)) { + upb_msg_field_iter i; + for(upb_msg_field_begin(&i, m); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { upb_fielddef *f = upb_msg_iter_field(&i); visit(r, UPB_UPCAST2(f), closure); } + upb_msg_oneof_iter o; + for(upb_msg_oneof_begin(&o, m); + !upb_msg_oneof_done(&o); + upb_msg_oneof_next(&o)) { + upb_oneofdef *f = upb_msg_iter_oneof(&o); + visit(r, UPB_UPCAST2(f), closure); + } } static void freemsg(upb_refcounted *r) { upb_msgdef *m = (upb_msgdef*)r; + upb_strtable_uninit(&m->ntoo); upb_strtable_uninit(&m->ntof); upb_inttable_uninit(&m->itof); upb_def_uninit(UPB_UPCAST(m)); @@ -1267,14 +1313,17 @@ upb_msgdef *upb_msgdef_new(const void *owner) { upb_msgdef *m = malloc(sizeof(*m)); if (!m) return NULL; if (!upb_def_init(UPB_UPCAST(m), UPB_DEF_MSG, &vtbl, owner)) goto err2; - if (!upb_inttable_init(&m->itof, UPB_CTYPE_PTR)) goto err2; - if (!upb_strtable_init(&m->ntof, UPB_CTYPE_PTR)) goto err1; + if (!upb_inttable_init(&m->itof, UPB_CTYPE_PTR)) goto err3; + if (!upb_strtable_init(&m->ntof, UPB_CTYPE_PTR)) goto err2; + if (!upb_strtable_init(&m->ntoo, UPB_CTYPE_PTR)) goto err1; m->map_entry = false; return m; err1: - upb_inttable_uninit(&m->itof); + upb_strtable_uninit(&m->ntof); err2: + upb_inttable_uninit(&m->itof); +err3: free(m); return NULL; } @@ -1286,14 +1335,28 @@ upb_msgdef *upb_msgdef_dup(const upb_msgdef *m, const void *owner) { upb_def_fullname(UPB_UPCAST(m)), NULL); newm->map_entry = m->map_entry; UPB_ASSERT_VAR(ok, ok); - upb_msg_iter i; - for(upb_msg_begin(&i, m); !upb_msg_done(&i); upb_msg_next(&i)) { + upb_msg_field_iter i; + for(upb_msg_field_begin(&i, m); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { upb_fielddef *f = upb_fielddef_dup(upb_msg_iter_field(&i), &f); + // Fields in oneofs are dup'd below. + if (upb_fielddef_containingoneof(f)) continue; if (!f || !upb_msgdef_addfield(newm, f, &f, NULL)) { upb_msgdef_unref(newm, owner); return NULL; } } + upb_msg_oneof_iter o; + for(upb_msg_oneof_begin(&o, m); + !upb_msg_oneof_done(&o); + upb_msg_oneof_next(&o)) { + upb_oneofdef *f = upb_oneofdef_dup(upb_msg_iter_oneof(&o), &f); + if (!f || !upb_msgdef_addoneof(newm, f, &f, NULL)) { + upb_msgdef_unref(newm, owner); + return NULL; + } + } return newm; } @@ -1332,6 +1395,35 @@ bool upb_msgdef_setfullname(upb_msgdef *m, const char *fullname, return upb_def_setfullname(UPB_UPCAST(m), fullname, s); } +// Helper: check that the field |f| is safe to add to msgdef |m|. Set an error +// on status |s| and return false if not. +static bool check_field_add(const upb_msgdef *m, const upb_fielddef *f, + upb_status *s) { + if (upb_fielddef_containingtype(f) != NULL) { + upb_status_seterrmsg(s, "fielddef already belongs to a message"); + return false; + } else if (upb_fielddef_name(f) == NULL || upb_fielddef_number(f) == 0) { + upb_status_seterrmsg(s, "field name or number were not set"); + return false; + } else if (upb_msgdef_ntofz(m, upb_fielddef_name(f)) || + upb_msgdef_itof(m, upb_fielddef_number(f))) { + upb_status_seterrmsg(s, "duplicate field name or number for field"); + return false; + } + return true; +} + +static void add_field(upb_msgdef *m, upb_fielddef *f, const void *ref_donor) { + release_containingtype(f); + f->msg.def = m; + f->msg_is_symbolic = false; + upb_inttable_insert(&m->itof, upb_fielddef_number(f), upb_value_ptr(f)); + upb_strtable_insert(&m->ntof, upb_fielddef_name(f), upb_value_ptr(f)); + upb_ref2(f, m); + upb_ref2(m, f); + if (ref_donor) upb_fielddef_unref(f, ref_donor); +} + bool upb_msgdef_addfield(upb_msgdef *m, upb_fielddef *f, const void *ref_donor, upb_status *s) { // TODO: extensions need to have a separate namespace, because proto2 allows a @@ -1345,28 +1437,65 @@ bool upb_msgdef_addfield(upb_msgdef *m, upb_fielddef *f, const void *ref_donor, // We also need to validate that the field number is in an extension range iff // it is an extension. + // This method is idempotent. Check if |f| is already part of this msgdef and + // return immediately if so. + if (upb_fielddef_containingtype(f) == m) { + return true; + } + // Check constraints for all fields before performing any action. - if (upb_fielddef_containingtype(f) != NULL) { - upb_status_seterrmsg(s, "fielddef already belongs to a message"); + if (!check_field_add(m, f, s)) { return false; - } else if (upb_fielddef_name(f) == NULL || upb_fielddef_number(f) == 0) { - upb_status_seterrmsg(s, "field name or number were not set"); - return false; - } else if(upb_msgdef_itof(m, upb_fielddef_number(f)) || - upb_msgdef_ntofz(m, upb_fielddef_name(f))) { - upb_status_seterrmsg(s, "duplicate field name or number"); + } else if (upb_fielddef_containingoneof(f) != NULL) { + // Fields in a oneof can only be added by adding the oneof to the msgdef. + upb_status_seterrmsg(s, "fielddef is part of a oneof"); return false; } // Constraint checks ok, perform the action. - release_containingtype(f); - f->msg.def = m; - f->msg_is_symbolic = false; - upb_inttable_insert(&m->itof, upb_fielddef_number(f), upb_value_ptr(f)); - upb_strtable_insert(&m->ntof, upb_fielddef_name(f), upb_value_ptr(f)); - upb_ref2(f, m); - upb_ref2(m, f); - if (ref_donor) upb_fielddef_unref(f, ref_donor); + add_field(m, f, ref_donor); + return true; +} + +bool upb_msgdef_addoneof(upb_msgdef *m, upb_oneofdef *o, const void *ref_donor, + upb_status *s) { + // Check various conditions that would prevent this oneof from being added. + if (upb_oneofdef_containingtype(o)) { + upb_status_seterrmsg(s, "oneofdef already belongs to a message"); + return false; + } else if (upb_oneofdef_name(o) == NULL) { + upb_status_seterrmsg(s, "oneofdef name was not set"); + return false; + } else if (upb_msgdef_ntooz(m, upb_oneofdef_name(o))) { + upb_status_seterrmsg(s, "duplicate oneof name"); + return false; + } + + // Check that all of the oneof's fields do not conflict with names or numbers + // of fields already in the message. + upb_oneof_iter it; + for (upb_oneof_begin(&it, o); !upb_oneof_done(&it); upb_oneof_next(&it)) { + const upb_fielddef *f = upb_oneof_iter_field(&it); + if (!check_field_add(m, f, s)) { + return false; + } + } + + // Everything checks out -- commit now. + + // Add oneof itself first. + o->parent = m; + upb_strtable_insert(&m->ntoo, upb_oneofdef_name(o), upb_value_ptr(o)); + upb_ref2(o, m); + upb_ref2(m, o); + + // Add each field of the oneof directly to the msgdef. + for (upb_oneof_begin(&it, o); !upb_oneof_done(&it); upb_oneof_next(&it)) { + upb_fielddef *f = upb_oneof_iter_field(&it); + add_field(m, f, NULL); + } + + if (ref_donor) upb_oneofdef_unref(o, ref_donor); return true; } @@ -1384,10 +1513,21 @@ const upb_fielddef *upb_msgdef_ntof(const upb_msgdef *m, const char *name, upb_value_getptr(val) : NULL; } +const upb_oneofdef *upb_msgdef_ntoo(const upb_msgdef *m, const char *name, + size_t len) { + upb_value val; + return upb_strtable_lookup2(&m->ntoo, name, len, &val) ? + upb_value_getptr(val) : NULL; +} + int upb_msgdef_numfields(const upb_msgdef *m) { return upb_strtable_count(&m->ntof); } +int upb_msgdef_numoneofs(const upb_msgdef *m) { + return upb_strtable_count(&m->ntoo); +} + void upb_msgdef_setmapentry(upb_msgdef *m, bool map_entry) { assert(!upb_msgdef_isfrozen(m)); m->map_entry = map_entry; @@ -1397,19 +1537,246 @@ bool upb_msgdef_mapentry(const upb_msgdef *m) { return m->map_entry; } -void upb_msg_begin(upb_msg_iter *iter, const upb_msgdef *m) { +void upb_msg_field_begin(upb_msg_field_iter *iter, const upb_msgdef *m) { upb_inttable_begin(iter, &m->itof); } -void upb_msg_next(upb_msg_iter *iter) { upb_inttable_next(iter); } +void upb_msg_field_next(upb_msg_field_iter *iter) { upb_inttable_next(iter); } + +bool upb_msg_field_done(const upb_msg_field_iter *iter) { + return upb_inttable_done(iter); +} + +upb_fielddef *upb_msg_iter_field(const upb_msg_field_iter *iter) { + return (upb_fielddef*)upb_value_getptr(upb_inttable_iter_value(iter)); +} + +void upb_msg_field_iter_setdone(upb_msg_field_iter *iter) { + upb_inttable_iter_setdone(iter); +} + +void upb_msg_oneof_begin(upb_msg_oneof_iter *iter, const upb_msgdef *m) { + upb_strtable_begin(iter, &m->ntoo); +} + +void upb_msg_oneof_next(upb_msg_oneof_iter *iter) { upb_strtable_next(iter); } + +bool upb_msg_oneof_done(const upb_msg_oneof_iter *iter) { + return upb_strtable_done(iter); +} + +upb_oneofdef *upb_msg_iter_oneof(const upb_msg_oneof_iter *iter) { + return (upb_oneofdef*)upb_value_getptr(upb_strtable_iter_value(iter)); +} + +void upb_msg_oneof_iter_setdone(upb_msg_oneof_iter *iter) { + upb_strtable_iter_setdone(iter); +} + +/* upb_oneofdef ***************************************************************/ + +static void visitoneof(const upb_refcounted *r, upb_refcounted_visit *visit, + void *closure) { + const upb_oneofdef *o = (const upb_oneofdef*)r; + upb_oneof_iter i; + for (upb_oneof_begin(&i, o); !upb_oneof_done(&i); upb_oneof_next(&i)) { + const upb_fielddef *f = upb_oneof_iter_field(&i); + visit(r, UPB_UPCAST2(f), closure); + } + if (o->parent) { + visit(r, UPB_UPCAST2(o->parent), closure); + } +} + +static void freeoneof(upb_refcounted *r) { + upb_oneofdef *o = (upb_oneofdef*)r; + upb_strtable_uninit(&o->ntof); + upb_inttable_uninit(&o->itof); + upb_def_uninit(UPB_UPCAST(o)); + free(o); +} + +upb_oneofdef *upb_oneofdef_new(const void *owner) { + static const struct upb_refcounted_vtbl vtbl = {visitoneof, freeoneof}; + upb_oneofdef *o = malloc(sizeof(*o)); + o->parent = NULL; + if (!o) return NULL; + if (!upb_def_init(UPB_UPCAST(o), UPB_DEF_ONEOF, &vtbl, owner)) goto err2; + if (!upb_inttable_init(&o->itof, UPB_CTYPE_PTR)) goto err2; + if (!upb_strtable_init(&o->ntof, UPB_CTYPE_PTR)) goto err1; + return o; + +err1: + upb_inttable_uninit(&o->itof); +err2: + free(o); + return NULL; +} + +upb_oneofdef *upb_oneofdef_dup(const upb_oneofdef *o, const void *owner) { + upb_oneofdef *newo = upb_oneofdef_new(owner); + if (!newo) return NULL; + bool ok = upb_def_setfullname(UPB_UPCAST(newo), + upb_def_fullname(UPB_UPCAST(o)), NULL); + UPB_ASSERT_VAR(ok, ok); + upb_oneof_iter i; + for (upb_oneof_begin(&i, o); !upb_oneof_done(&i); upb_oneof_next(&i)) { + upb_fielddef *f = upb_fielddef_dup(upb_oneof_iter_field(&i), &f); + if (!f || !upb_oneofdef_addfield(newo, f, &f, NULL)) { + upb_oneofdef_unref(newo, owner); + return NULL; + } + } + return newo; +} + +bool upb_oneofdef_isfrozen(const upb_oneofdef *o) { + return upb_def_isfrozen(UPB_UPCAST(o)); +} + +void upb_oneofdef_ref(const upb_oneofdef *o, const void *owner) { + upb_def_ref(UPB_UPCAST(o), owner); +} + +void upb_oneofdef_unref(const upb_oneofdef *o, const void *owner) { + upb_def_unref(UPB_UPCAST(o), owner); +} + +void upb_oneofdef_donateref(const upb_oneofdef *o, const void *from, + const void *to) { + upb_def_donateref(UPB_UPCAST(o), from, to); +} + +void upb_oneofdef_checkref(const upb_oneofdef *o, const void *owner) { + upb_def_checkref(UPB_UPCAST(o), owner); +} + +const char *upb_oneofdef_name(const upb_oneofdef *o) { + return upb_def_fullname(UPB_UPCAST(o)); +} + +bool upb_oneofdef_setname(upb_oneofdef *o, const char *fullname, + upb_status *s) { + if (upb_oneofdef_containingtype(o)) { + upb_status_seterrmsg(s, "oneof already added to a message"); + return false; + } + return upb_def_setfullname(UPB_UPCAST(o), fullname, s); +} + +const upb_msgdef *upb_oneofdef_containingtype(const upb_oneofdef *o) { + return o->parent; +} + +int upb_oneofdef_numfields(const upb_oneofdef *o) { + return upb_strtable_count(&o->ntof); +} + +bool upb_oneofdef_addfield(upb_oneofdef *o, upb_fielddef *f, + const void *ref_donor, + upb_status *s) { + assert(!upb_oneofdef_isfrozen(o)); + assert(!o->parent || !upb_msgdef_isfrozen(o->parent)); + + // This method is idempotent. Check if |f| is already part of this oneofdef + // and return immediately if so. + if (upb_fielddef_containingoneof(f) == o) { + return true; + } + + // The field must have an OPTIONAL label. + if (upb_fielddef_label(f) != UPB_LABEL_OPTIONAL) { + upb_status_seterrmsg(s, "fields in oneof must have OPTIONAL label"); + return false; + } + + // Check that no field with this name or number exists already in the oneof. + // Also check that the field is not already part of a oneof. + if (upb_fielddef_name(f) == NULL || upb_fielddef_number(f) == 0) { + upb_status_seterrmsg(s, "field name or number were not set"); + return false; + } else if (upb_oneofdef_itof(o, upb_fielddef_number(f)) || + upb_oneofdef_ntofz(o, upb_fielddef_name(f))) { + upb_status_seterrmsg(s, "duplicate field name or number"); + return false; + } else if (upb_fielddef_containingoneof(f) != NULL) { + upb_status_seterrmsg(s, "fielddef already belongs to a oneof"); + return false; + } + + // We allow adding a field to the oneof either if the field is not part of a + // msgdef, or if it is and we are also part of the same msgdef. + if (o->parent == NULL) { + // If we're not in a msgdef, the field cannot be either. Otherwise we would + // need to magically add this oneof to a msgdef to remain consistent, which + // is surprising behavior. + if (upb_fielddef_containingtype(f) != NULL) { + upb_status_seterrmsg(s, "fielddef already belongs to a message, but " + "oneof does not"); + return false; + } + } else { + // If we're in a msgdef, the user can add fields that either aren't in any + // msgdef (in which case they're added to our msgdef) or already a part of + // our msgdef. + if (upb_fielddef_containingtype(f) != NULL && + upb_fielddef_containingtype(f) != o->parent) { + upb_status_seterrmsg(s, "fielddef belongs to a different message " + "than oneof"); + return false; + } + } + + // Commit phase. First add the field to our parent msgdef, if any, because + // that may fail; then add the field to our own tables. + + if (o->parent != NULL && upb_fielddef_containingtype(f) == NULL) { + if (!upb_msgdef_addfield((upb_msgdef*)o->parent, f, NULL, s)) { + return false; + } + } + + release_containingtype(f); + f->oneof = o; + upb_inttable_insert(&o->itof, upb_fielddef_number(f), upb_value_ptr(f)); + upb_strtable_insert(&o->ntof, upb_fielddef_name(f), upb_value_ptr(f)); + upb_ref2(f, o); + upb_ref2(o, f); + if (ref_donor) upb_fielddef_unref(f, ref_donor); + + return true; +} + +const upb_fielddef *upb_oneofdef_ntof(const upb_oneofdef *o, + const char *name, size_t length) { + upb_value val; + return upb_strtable_lookup2(&o->ntof, name, length, &val) ? + upb_value_getptr(val) : NULL; +} + +const upb_fielddef *upb_oneofdef_itof(const upb_oneofdef *o, uint32_t num) { + upb_value val; + return upb_inttable_lookup32(&o->itof, num, &val) ? + upb_value_getptr(val) : NULL; +} + +void upb_oneof_begin(upb_oneof_iter *iter, const upb_oneofdef *o) { + upb_inttable_begin(iter, &o->itof); +} + +void upb_oneof_next(upb_oneof_iter *iter) { + upb_inttable_next(iter); +} -bool upb_msg_done(const upb_msg_iter *iter) { return upb_inttable_done(iter); } +bool upb_oneof_done(upb_oneof_iter *iter) { + return upb_inttable_done(iter); +} -upb_fielddef *upb_msg_iter_field(const upb_msg_iter *iter) { +upb_fielddef *upb_oneof_iter_field(const upb_oneof_iter *iter) { return (upb_fielddef*)upb_value_getptr(upb_inttable_iter_value(iter)); } -void upb_msg_iter_setdone(upb_msg_iter *iter) { +void upb_oneof_iter_setdone(upb_oneof_iter *iter) { upb_inttable_iter_setdone(iter); } /* @@ -1452,8 +1819,10 @@ static void freehandlers(upb_refcounted *r) { static void visithandlers(const upb_refcounted *r, upb_refcounted_visit *visit, void *closure) { const upb_handlers *h = (const upb_handlers*)r; - upb_msg_iter i; - for(upb_msg_begin(&i, h->msg); !upb_msg_done(&i); upb_msg_next(&i)) { + upb_msg_field_iter i; + for(upb_msg_field_begin(&i, h->msg); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { upb_fielddef *f = upb_msg_iter_field(&i); if (!upb_fielddef_issubmsg(f)) continue; const upb_handlers *sub = upb_handlers_getsubhandlers(h, f); @@ -1482,8 +1851,10 @@ static upb_handlers *newformsg(const upb_msgdef *m, const void *owner, // For each submessage field, get or create a handlers object and set it as // the subhandlers. - upb_msg_iter i; - for(upb_msg_begin(&i, m); !upb_msg_done(&i); upb_msg_next(&i)) { + upb_msg_field_iter i; + for(upb_msg_field_begin(&i, m); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { upb_fielddef *f = upb_msg_iter_field(&i); if (!upb_fielddef_issubmsg(f)) continue; @@ -1840,8 +2211,10 @@ bool upb_handlers_freeze(upb_handlers *const*handlers, int n, upb_status *s) { // Check that there are no closure mismatches due to missing Start* handlers // or subhandlers with different type-level types. - upb_msg_iter j; - for(upb_msg_begin(&j, h->msg); !upb_msg_done(&j); upb_msg_next(&j)) { + upb_msg_field_iter j; + for(upb_msg_field_begin(&j, h->msg); + !upb_msg_field_done(&j); + upb_msg_field_next(&j)) { const upb_fielddef *f = upb_msg_iter_field(&j); if (upb_fielddef_isseq(f)) { @@ -3114,8 +3487,10 @@ static bool upb_resolve_dfs(const upb_def *def, upb_strtable *addtab, // For messages, continue the recursion by visiting all subdefs. const upb_msgdef *m = upb_dyncast_msgdef(def); if (m) { - upb_msg_iter i; - for(upb_msg_begin(&i, m); !upb_msg_done(&i); upb_msg_next(&i)) { + upb_msg_field_iter i; + for(upb_msg_field_begin(&i, m); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { upb_fielddef *f = upb_msg_iter_field(&i); if (!upb_fielddef_hassubdef(f)) continue; // |= to avoid short-circuit; we need its side-effects. @@ -3268,8 +3643,10 @@ bool upb_symtab_add(upb_symtab *s, upb_def *const*defs, int n, void *ref_donor, // Type names are resolved relative to the message in which they appear. const char *base = upb_msgdef_fullname(m); - upb_msg_iter j; - for(upb_msg_begin(&j, m); !upb_msg_done(&j); upb_msg_next(&j)) { + upb_msg_field_iter j; + for(upb_msg_field_begin(&j, m); + !upb_msg_field_done(&j); + upb_msg_field_next(&j)) { upb_fielddef *f = upb_msg_iter_field(&j); const char *name = upb_fielddef_subdefname(f); if (name && !upb_fielddef_subdef(f)) { @@ -6462,8 +6839,10 @@ static void compile_method(compiler *c, upb_pbdecodermethod *method) { putsel(c, OP_STARTMSG, UPB_STARTMSG_SELECTOR, h); label(c, LABEL_FIELD); uint32_t* start_pc = c->pc; - upb_msg_iter i; - for(upb_msg_begin(&i, md); !upb_msg_done(&i); upb_msg_next(&i)) { + upb_msg_field_iter i; + for(upb_msg_field_begin(&i, md); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { const upb_fielddef *f = upb_msg_iter_field(&i); upb_fieldtype_t type = upb_fielddef_type(f); @@ -6513,9 +6892,11 @@ static void find_methods(compiler *c, const upb_handlers *h) { newmethod(h, c->group); // Find submethods. - upb_msg_iter i; + upb_msg_field_iter i; const upb_msgdef *md = upb_handlers_msgdef(h); - for(upb_msg_begin(&i, md); !upb_msg_done(&i); upb_msg_next(&i)) { + for(upb_msg_field_begin(&i, md); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { const upb_fielddef *f = upb_msg_iter_field(&i); const upb_handlers *sub_h; if (upb_fielddef_type(f) == UPB_TYPE_MESSAGE && @@ -6557,7 +6938,7 @@ static void set_bytecode_handlers(mgroup *g) { } -/* JIT setup. ******************************************************************/ +/* JIT setup. *****************************************************************/ #ifdef UPB_USE_JIT_X64 @@ -7980,8 +8361,10 @@ static void newhandlers_callback(const void *closure, upb_handlers *h) { upb_handlers_setendmsg(h, endmsg, NULL); const upb_msgdef *m = upb_handlers_msgdef(h); - upb_msg_iter i; - for(upb_msg_begin(&i, m); !upb_msg_done(&i); upb_msg_next(&i)) { + upb_msg_field_iter i; + for(upb_msg_field_begin(&i, m); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { const upb_fielddef *f = upb_msg_iter_field(&i); bool packed = upb_fielddef_isseq(f) && upb_fielddef_isprimitive(f) && upb_fielddef_packed(f); @@ -8443,8 +8826,10 @@ static void onmreg(const void *c, upb_handlers *h) { upb_handlers_setstartmsg(h, textprinter_startmsg, NULL); upb_handlers_setendmsg(h, textprinter_endmsg, NULL); - upb_msg_iter i; - for(upb_msg_begin(&i, m); !upb_msg_done(&i); upb_msg_next(&i)) { + upb_msg_field_iter i; + for(upb_msg_field_begin(&i, m); + !upb_msg_field_done(&i); + upb_msg_field_next(&i)) { upb_fielddef *f = upb_msg_iter_field(&i); upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER; upb_handlerattr_sethandlerdata(&attr, f); @@ -8857,6 +9242,7 @@ badpadding: // the true value in a contiguous buffer. static void assert_accumulate_empty(upb_json_parser *p) { + UPB_UNUSED(p); assert(p->accumulated == NULL); assert(p->accumulated_len == 0); } @@ -9141,11 +9527,17 @@ static void start_number(upb_json_parser *p, const char *ptr) { capture_begin(p, ptr); } +static bool parse_number(upb_json_parser *p); + static bool end_number(upb_json_parser *p, const char *ptr) { if (!capture_end(p, ptr)) { return false; } + return parse_number(p); +} + +static bool parse_number(upb_json_parser *p) { // strtol() and friends unfortunately do not support specifying the length of // the input string, so we need to force a copy into a NULL-terminated buffer. if (!multipart_text(p, "\0", 1, false)) { @@ -9155,8 +9547,8 @@ static bool end_number(upb_json_parser *p, const char *ptr) { size_t len; const char *buf = accumulate_getptr(p, &len); const char *myend = buf + len - 1; // One for NULL. - char *end; + char *end; switch (upb_fielddef_type(p->top->f)) { case UPB_TYPE_ENUM: case UPB_TYPE_INT32: { @@ -9212,6 +9604,7 @@ static bool end_number(upb_json_parser *p, const char *ptr) { } multipart_end(p); + return true; err: @@ -9230,6 +9623,7 @@ static bool parser_putbool(upb_json_parser *p, bool val) { bool ok = upb_sink_putbool(&p->top->sink, parser_getsel(p), val); UPB_ASSERT_VAR(ok, ok); + return true; } @@ -9246,6 +9640,8 @@ static bool start_stringval(upb_json_parser *p) { upb_sink_startstr(&p->top->sink, sel, 0, &inner->sink); inner->m = p->top->m; inner->f = p->top->f; + inner->is_map = false; + inner->is_mapentry = false; p->top = inner; if (upb_fielddef_type(p->top->f) == UPB_TYPE_STRING) { @@ -9323,6 +9719,7 @@ static bool end_stringval(upb_json_parser *p) { } multipart_end(p); + return ok; } @@ -9331,54 +9728,217 @@ static void start_member(upb_json_parser *p) { multipart_startaccum(p); } -static bool end_member(upb_json_parser *p) { - assert(!p->top->f); +// Helper: invoked during parse_mapentry() to emit the mapentry message's key +// field based on the current contents of the accumulate buffer. +static bool parse_mapentry_key(upb_json_parser *p) { + size_t len; const char *buf = accumulate_getptr(p, &len); - const upb_fielddef *f = upb_msgdef_ntof(p->top->m, buf, len); + // Emit the key field. We do a bit of ad-hoc parsing here because the + // parser state machine has already decided that this is a string field + // name, and we are reinterpreting it as some arbitrary key type. In + // particular, integer and bool keys are quoted, so we need to parse the + // quoted string contents here. - if (!f) { - // TODO(haberman): Ignore unknown fields if requested/configured to do so. - upb_status_seterrf(p->status, "No such field: %.*s\n", (int)len, buf); + p->top->f = upb_msgdef_itof(p->top->m, UPB_MAPENTRY_KEY); + if (p->top->f == NULL) { + upb_status_seterrmsg(p->status, "mapentry message has no key"); return false; } + switch (upb_fielddef_type(p->top->f)) { + case UPB_TYPE_INT32: + case UPB_TYPE_INT64: + case UPB_TYPE_UINT32: + case UPB_TYPE_UINT64: + // Invoke end_number. The accum buffer has the number's text already. + if (!parse_number(p)) { + return false; + } + break; + case UPB_TYPE_BOOL: + if (len == 4 && !strncmp(buf, "true", 4)) { + if (!parser_putbool(p, true)) { + return false; + } + } else if (len == 5 && !strncmp(buf, "false", 5)) { + if (!parser_putbool(p, false)) { + return false; + } + } else { + upb_status_seterrmsg(p->status, + "Map bool key not 'true' or 'false'"); + return false; + } + multipart_end(p); + break; + case UPB_TYPE_STRING: + case UPB_TYPE_BYTES: { + upb_sink subsink; + upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSTR); + upb_sink_startstr(&p->top->sink, sel, len, &subsink); + sel = getsel_for_handlertype(p, UPB_HANDLER_STRING); + upb_sink_putstring(&subsink, sel, buf, len, NULL); + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSTR); + upb_sink_endstr(&subsink, sel); + multipart_end(p); + break; + } + default: + upb_status_seterrmsg(p->status, "Invalid field type for map key"); + return false; + } - p->top->f = f; - multipart_end(p); + return true; +} + +// Helper: emit one map entry (as a submessage in the map field sequence). This +// is invoked from end_membername(), at the end of the map entry's key string, +// with the map key in the accumulate buffer. It parses the key from that +// buffer, emits the handler calls to start the mapentry submessage (setting up +// its subframe in the process), and sets up state in the subframe so that the +// value parser (invoked next) will emit the mapentry's value field and then +// end the mapentry message. + +static bool handle_mapentry(upb_json_parser *p) { + // Map entry: p->top->sink is the seq frame, so we need to start a frame + // for the mapentry itself, and then set |f| in that frame so that the map + // value field is parsed, and also set a flag to end the frame after the + // map-entry value is parsed. + if (!check_stack(p)) return false; + + const upb_fielddef *mapfield = p->top->mapfield; + const upb_msgdef *mapentrymsg = upb_fielddef_msgsubdef(mapfield); + + upb_jsonparser_frame *inner = p->top + 1; + p->top->f = mapfield; + upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSUBMSG); + upb_sink_startsubmsg(&p->top->sink, sel, &inner->sink); + inner->m = mapentrymsg; + inner->mapfield = mapfield; + inner->is_map = false; + + // Don't set this to true *yet* -- we reuse parsing handlers below to push + // the key field value to the sink, and these handlers will pop the frame + // if they see is_mapentry (when invoked by the parser state machine, they + // would have just seen the map-entry value, not key). + inner->is_mapentry = false; + p->top = inner; + + // send STARTMSG in submsg frame. + upb_sink_startmsg(&p->top->sink); + + parse_mapentry_key(p); + + // Set up the value field to receive the map-entry value. + p->top->f = upb_msgdef_itof(p->top->m, UPB_MAPENTRY_VALUE); + p->top->is_mapentry = true; // set up to pop frame after value is parsed. + p->top->mapfield = mapfield; + if (p->top->f == NULL) { + upb_status_seterrmsg(p->status, "mapentry message has no value"); + return false; + } return true; } -static void clear_member(upb_json_parser *p) { p->top->f = NULL; } +static bool end_membername(upb_json_parser *p) { + assert(!p->top->f); + + if (p->top->is_map) { + return handle_mapentry(p); + } else { + size_t len; + const char *buf = accumulate_getptr(p, &len); + const upb_fielddef *f = upb_msgdef_ntof(p->top->m, buf, len); + + if (!f) { + // TODO(haberman): Ignore unknown fields if requested/configured to do so. + upb_status_seterrf(p->status, "No such field: %.*s\n", (int)len, buf); + return false; + } + + p->top->f = f; + multipart_end(p); + + return true; + } +} + +static void end_member(upb_json_parser *p) { + // If we just parsed a map-entry value, end that frame too. + if (p->top->is_mapentry) { + assert(p->top > p->stack); + // send ENDMSG on submsg. + upb_status s = UPB_STATUS_INIT; + upb_sink_endmsg(&p->top->sink, &s); + const upb_fielddef* mapfield = p->top->mapfield; + + // send ENDSUBMSG in repeated-field-of-mapentries frame. + p->top--; + upb_selector_t sel; + bool ok = upb_handlers_getselector(mapfield, + UPB_HANDLER_ENDSUBMSG, &sel); + UPB_ASSERT_VAR(ok, ok); + upb_sink_endsubmsg(&p->top->sink, sel); + } + + p->top->f = NULL; +} static bool start_subobject(upb_json_parser *p) { assert(p->top->f); - if (!upb_fielddef_issubmsg(p->top->f)) { + if (upb_fielddef_ismap(p->top->f)) { + // Beginning of a map. Start a new parser frame in a repeated-field + // context. + if (!check_stack(p)) return false; + + upb_jsonparser_frame *inner = p->top + 1; + upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSEQ); + upb_sink_startseq(&p->top->sink, sel, &inner->sink); + inner->m = upb_fielddef_msgsubdef(p->top->f); + inner->mapfield = p->top->f; + inner->f = NULL; + inner->is_map = true; + inner->is_mapentry = false; + p->top = inner; + + return true; + } else if (upb_fielddef_issubmsg(p->top->f)) { + // Beginning of a subobject. Start a new parser frame in the submsg + // context. + if (!check_stack(p)) return false; + + upb_jsonparser_frame *inner = p->top + 1; + + upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSUBMSG); + upb_sink_startsubmsg(&p->top->sink, sel, &inner->sink); + inner->m = upb_fielddef_msgsubdef(p->top->f); + inner->f = NULL; + inner->is_map = false; + inner->is_mapentry = false; + p->top = inner; + + return true; + } else { upb_status_seterrf(p->status, "Object specified for non-message/group field: %s", upb_fielddef_name(p->top->f)); return false; } - - if (!check_stack(p)) return false; - - upb_jsonparser_frame *inner = p->top + 1; - - upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSUBMSG); - upb_sink_startsubmsg(&p->top->sink, sel, &inner->sink); - inner->m = upb_fielddef_msgsubdef(p->top->f); - inner->f = NULL; - p->top = inner; - - return true; } static void end_subobject(upb_json_parser *p) { - p->top--; - upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSUBMSG); - upb_sink_endsubmsg(&p->top->sink, sel); + if (p->top->is_map) { + p->top--; + upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSEQ); + upb_sink_endseq(&p->top->sink, sel); + } else { + p->top--; + upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSUBMSG); + upb_sink_endsubmsg(&p->top->sink, sel); + } } static bool start_array(upb_json_parser *p) { @@ -9398,6 +9958,8 @@ static bool start_array(upb_json_parser *p) { upb_sink_startseq(&p->top->sink, sel, &inner->sink); inner->m = p->top->m; inner->f = p->top->f; + inner->is_map = false; + inner->is_mapentry = false; p->top = inner; return true; @@ -9412,12 +9974,16 @@ static void end_array(upb_json_parser *p) { } static void start_object(upb_json_parser *p) { - upb_sink_startmsg(&p->top->sink); + if (!p->top->is_map) { + upb_sink_startmsg(&p->top->sink); + } } static void end_object(upb_json_parser *p) { - upb_status status; - upb_sink_endmsg(&p->top->sink, &status); + if (!p->top->is_map) { + upb_status status; + upb_sink_endmsg(&p->top->sink, &status); + } } @@ -9442,11 +10008,11 @@ static void end_object(upb_json_parser *p) { // final state once, when the closing '"' is seen. -#line 904 "upb/json/parser.rl" +#line 1085 "upb/json/parser.rl" -#line 816 "upb/json/parser.c" +#line 997 "upb/json/parser.c" static const char _json_actions[] = { 0, 1, 0, 1, 2, 1, 3, 1, 5, 1, 6, 1, 7, 1, 8, 1, @@ -9597,7 +10163,7 @@ static const int json_en_value_machine = 27; static const int json_en_main = 1; -#line 907 "upb/json/parser.rl" +#line 1088 "upb/json/parser.rl" size_t parse(void *closure, const void *hd, const char *buf, size_t size, const upb_bufhandle *handle) { @@ -9617,7 +10183,7 @@ size_t parse(void *closure, const void *hd, const char *buf, size_t size, capture_resume(parser, buf); -#line 987 "upb/json/parser.c" +#line 1168 "upb/json/parser.c" { int _klen; unsigned int _trans; @@ -9692,118 +10258,118 @@ _match: switch ( *_acts++ ) { case 0: -#line 819 "upb/json/parser.rl" +#line 1000 "upb/json/parser.rl" { p--; {cs = stack[--top]; goto _again;} } break; case 1: -#line 820 "upb/json/parser.rl" +#line 1001 "upb/json/parser.rl" { p--; {stack[top++] = cs; cs = 10; goto _again;} } break; case 2: -#line 824 "upb/json/parser.rl" +#line 1005 "upb/json/parser.rl" { start_text(parser, p); } break; case 3: -#line 825 "upb/json/parser.rl" +#line 1006 "upb/json/parser.rl" { CHECK_RETURN_TOP(end_text(parser, p)); } break; case 4: -#line 831 "upb/json/parser.rl" +#line 1012 "upb/json/parser.rl" { start_hex(parser); } break; case 5: -#line 832 "upb/json/parser.rl" +#line 1013 "upb/json/parser.rl" { hexdigit(parser, p); } break; case 6: -#line 833 "upb/json/parser.rl" +#line 1014 "upb/json/parser.rl" { CHECK_RETURN_TOP(end_hex(parser)); } break; case 7: -#line 839 "upb/json/parser.rl" +#line 1020 "upb/json/parser.rl" { CHECK_RETURN_TOP(escape(parser, p)); } break; case 8: -#line 845 "upb/json/parser.rl" +#line 1026 "upb/json/parser.rl" { p--; {cs = stack[--top]; goto _again;} } break; case 9: -#line 848 "upb/json/parser.rl" +#line 1029 "upb/json/parser.rl" { {stack[top++] = cs; cs = 19; goto _again;} } break; case 10: -#line 850 "upb/json/parser.rl" +#line 1031 "upb/json/parser.rl" { p--; {stack[top++] = cs; cs = 27; goto _again;} } break; case 11: -#line 855 "upb/json/parser.rl" +#line 1036 "upb/json/parser.rl" { start_member(parser); } break; case 12: -#line 856 "upb/json/parser.rl" - { CHECK_RETURN_TOP(end_member(parser)); } +#line 1037 "upb/json/parser.rl" + { CHECK_RETURN_TOP(end_membername(parser)); } break; case 13: -#line 859 "upb/json/parser.rl" - { clear_member(parser); } +#line 1040 "upb/json/parser.rl" + { end_member(parser); } break; case 14: -#line 865 "upb/json/parser.rl" +#line 1046 "upb/json/parser.rl" { start_object(parser); } break; case 15: -#line 868 "upb/json/parser.rl" +#line 1049 "upb/json/parser.rl" { end_object(parser); } break; case 16: -#line 874 "upb/json/parser.rl" +#line 1055 "upb/json/parser.rl" { CHECK_RETURN_TOP(start_array(parser)); } break; case 17: -#line 878 "upb/json/parser.rl" +#line 1059 "upb/json/parser.rl" { end_array(parser); } break; case 18: -#line 883 "upb/json/parser.rl" +#line 1064 "upb/json/parser.rl" { start_number(parser, p); } break; case 19: -#line 884 "upb/json/parser.rl" +#line 1065 "upb/json/parser.rl" { CHECK_RETURN_TOP(end_number(parser, p)); } break; case 20: -#line 886 "upb/json/parser.rl" +#line 1067 "upb/json/parser.rl" { CHECK_RETURN_TOP(start_stringval(parser)); } break; case 21: -#line 887 "upb/json/parser.rl" +#line 1068 "upb/json/parser.rl" { CHECK_RETURN_TOP(end_stringval(parser)); } break; case 22: -#line 889 "upb/json/parser.rl" +#line 1070 "upb/json/parser.rl" { CHECK_RETURN_TOP(parser_putbool(parser, true)); } break; case 23: -#line 891 "upb/json/parser.rl" +#line 1072 "upb/json/parser.rl" { CHECK_RETURN_TOP(parser_putbool(parser, false)); } break; case 24: -#line 893 "upb/json/parser.rl" +#line 1074 "upb/json/parser.rl" { /* null value */ } break; case 25: -#line 895 "upb/json/parser.rl" +#line 1076 "upb/json/parser.rl" { CHECK_RETURN_TOP(start_subobject(parser)); } break; case 26: -#line 896 "upb/json/parser.rl" +#line 1077 "upb/json/parser.rl" { end_subobject(parser); } break; case 27: -#line 901 "upb/json/parser.rl" +#line 1082 "upb/json/parser.rl" { p--; {cs = stack[--top]; goto _again;} } break; -#line 1173 "upb/json/parser.c" +#line 1354 "upb/json/parser.c" } } @@ -9816,7 +10382,7 @@ _again: _out: {} } -#line 926 "upb/json/parser.rl" +#line 1107 "upb/json/parser.rl" if (p != pe) { upb_status_seterrf(parser->status, "Parse error at %s\n", p); @@ -9860,18 +10426,20 @@ void upb_json_parser_uninit(upb_json_parser *p) { void upb_json_parser_reset(upb_json_parser *p) { p->top = p->stack; p->top->f = NULL; + p->top->is_map = false; + p->top->is_mapentry = false; int cs; int top; // Emit Ragel initialization of the parser. -#line 1235 "upb/json/parser.c" +#line 1418 "upb/json/parser.c" { cs = json_start; top = 0; } -#line 974 "upb/json/parser.rl" +#line 1157 "upb/json/parser.rl" p->current_state = cs; p->parser_top = top; accumulate_clear(p); @@ -10072,13 +10640,23 @@ static bool putkey(void *closure, const void *handler_data) { return true; \ } \ static bool repeated_##type(void *closure, const void *handler_data, \ - type val) { \ + type val) { \ upb_json_printer *p = closure; \ print_comma(p); \ CHK(put##type(closure, handler_data, val)); \ return true; \ } +#define TYPE_HANDLERS_MAPKEY(type, fmt_func) \ + static bool putmapkey_##type(void *closure, const void *handler_data, \ + type val) { \ + upb_json_printer *p = closure; \ + print_data(p, "\"", 1); \ + CHK(put##type(closure, handler_data, val)); \ + print_data(p, "\":", 2); \ + return true; \ + } + TYPE_HANDLERS(double, fmt_double); TYPE_HANDLERS(float, fmt_float); TYPE_HANDLERS(bool, fmt_bool); @@ -10087,7 +10665,15 @@ TYPE_HANDLERS(uint32_t, fmt_int64); TYPE_HANDLERS(int64_t, fmt_int64); TYPE_HANDLERS(uint64_t, fmt_uint64); +// double and float are not allowed to be map keys. +TYPE_HANDLERS_MAPKEY(bool, fmt_bool); +TYPE_HANDLERS_MAPKEY(int32_t, fmt_int64); +TYPE_HANDLERS_MAPKEY(uint32_t, fmt_int64); +TYPE_HANDLERS_MAPKEY(int64_t, fmt_int64); +TYPE_HANDLERS_MAPKEY(uint64_t, fmt_uint64); + #undef TYPE_HANDLERS +#undef TYPE_HANDLERS_MAPKEY typedef struct { void *keyname; @@ -10112,20 +10698,36 @@ static bool scalar_enum(void *closure, const void *handler_data, return true; } -static bool repeated_enum(void *closure, const void *handler_data, - int32_t val) { - const EnumHandlerData *hd = handler_data; - upb_json_printer *p = closure; - print_comma(p); - - const char *symbolic_name = upb_enumdef_iton(hd->enumdef, val); +static void print_enum_symbolic_name(upb_json_printer *p, + const upb_enumdef *def, + int32_t val) { + const char *symbolic_name = upb_enumdef_iton(def, val); if (symbolic_name) { print_data(p, "\"", 1); putstring(p, symbolic_name, strlen(symbolic_name)); print_data(p, "\"", 1); } else { - putint32_t(closure, NULL, val); + putint32_t(p, NULL, val); } +} + +static bool repeated_enum(void *closure, const void *handler_data, + int32_t val) { + const EnumHandlerData *hd = handler_data; + upb_json_printer *p = closure; + print_comma(p); + + print_enum_symbolic_name(p, hd->enumdef, val); + + return true; +} + +static bool mapvalue_enum(void *closure, const void *handler_data, + int32_t val) { + const EnumHandlerData *hd = handler_data; + upb_json_printer *p = closure; + + print_enum_symbolic_name(p, hd->enumdef, val); return true; } @@ -10141,25 +10743,35 @@ static void *repeated_startsubmsg(void *closure, const void *handler_data) { return closure; } -static bool startmap(void *closure, const void *handler_data) { +static void start_frame(upb_json_printer *p) { + p->depth_++; + p->first_elem_[p->depth_] = true; + print_data(p, "{", 1); +} + +static void end_frame(upb_json_printer *p) { + print_data(p, "}", 1); + p->depth_--; +} + +static bool printer_startmsg(void *closure, const void *handler_data) { UPB_UNUSED(handler_data); upb_json_printer *p = closure; - if (p->depth_++ == 0) { + if (p->depth_ == 0) { upb_bytessink_start(p->output_, 0, &p->subc_); } - p->first_elem_[p->depth_] = true; - print_data(p, "{", 1); + start_frame(p); return true; } -static bool endmap(void *closure, const void *handler_data, upb_status *s) { +static bool printer_endmsg(void *closure, const void *handler_data, upb_status *s) { UPB_UNUSED(handler_data); UPB_UNUSED(s); upb_json_printer *p = closure; - if (--p->depth_ == 0) { + end_frame(p); + if (p->depth_ == 0) { upb_bytessink_end(p->output_); } - print_data(p, "}", 1); return true; } @@ -10180,6 +10792,23 @@ static bool endseq(void *closure, const void *handler_data) { return true; } +static void *startmap(void *closure, const void *handler_data) { + upb_json_printer *p = closure; + CHK(putkey(closure, handler_data)); + p->depth_++; + p->first_elem_[p->depth_] = true; + print_data(p, "{", 1); + return closure; +} + +static bool endmap(void *closure, const void *handler_data) { + UPB_UNUSED(handler_data); + upb_json_printer *p = closure; + print_data(p, "}", 1); + p->depth_--; + return true; +} + static size_t putstr(void *closure, const void *handler_data, const char *str, size_t len, const upb_bufhandle *handle) { UPB_UNUSED(handler_data); @@ -10294,6 +10923,36 @@ static bool repeated_endstr(void *closure, const void *handler_data) { return true; } +static void *mapkeyval_startstr(void *closure, const void *handler_data, + size_t size_hint) { + UPB_UNUSED(handler_data); + UPB_UNUSED(size_hint); + upb_json_printer *p = closure; + print_data(p, "\"", 1); + return p; +} + +static size_t mapkey_str(void *closure, const void *handler_data, + const char *str, size_t len, + const upb_bufhandle *handle) { + CHK(putstr(closure, handler_data, str, len, handle)); + return len; +} + +static bool mapkey_endstr(void *closure, const void *handler_data) { + UPB_UNUSED(handler_data); + upb_json_printer *p = closure; + print_data(p, "\":", 2); + return true; +} + +static bool mapvalue_endstr(void *closure, const void *handler_data) { + UPB_UNUSED(handler_data); + upb_json_printer *p = closure; + print_data(p, "\"", 1); + return true; +} + static size_t scalar_bytes(void *closure, const void *handler_data, const char *str, size_t len, const upb_bufhandle *handle) { @@ -10311,31 +10970,161 @@ static size_t repeated_bytes(void *closure, const void *handler_data, return len; } -void printer_sethandlers(const void *closure, upb_handlers *h) { +static size_t mapkey_bytes(void *closure, const void *handler_data, + const char *str, size_t len, + const upb_bufhandle *handle) { + upb_json_printer *p = closure; + CHK(putbytes(closure, handler_data, str, len, handle)); + print_data(p, ":", 1); + return len; +} + +static void set_enum_hd(upb_handlers *h, + const upb_fielddef *f, + upb_handlerattr *attr) { + EnumHandlerData *hd = malloc(sizeof(EnumHandlerData)); + hd->enumdef = (const upb_enumdef *)upb_fielddef_subdef(f); + hd->keyname = newstrpc(h, f); + upb_handlers_addcleanup(h, hd, free); + upb_handlerattr_sethandlerdata(attr, hd); +} + +// Set up handlers for a mapentry submessage (i.e., an individual key/value pair +// in a map). +// +// TODO: Handle missing key, missing value, out-of-order key/value, or repeated +// key or value cases properly. The right way to do this is to allocate a +// temporary structure at the start of a mapentry submessage, store key and +// value data in it as key and value handlers are called, and then print the +// key/value pair once at the end of the submessage. If we don't do this, we +// should at least detect the case and throw an error. However, so far all of +// our sources that emit mapentry messages do so canonically (with one key +// field, and then one value field), so this is not a pressing concern at the +// moment. +void printer_sethandlers_mapentry(const void *closure, upb_handlers *h) { UPB_UNUSED(closure); + const upb_msgdef *md = upb_handlers_msgdef(h); + // A mapentry message is printed simply as '"key": value'. Rather than + // special-case key and value for every type below, we just handle both + // fields explicitly here. + const upb_fielddef* key_field = upb_msgdef_itof(md, UPB_MAPENTRY_KEY); + const upb_fielddef* value_field = upb_msgdef_itof(md, UPB_MAPENTRY_VALUE); + + upb_handlerattr empty_attr = UPB_HANDLERATTR_INITIALIZER; + + switch (upb_fielddef_type(key_field)) { + case UPB_TYPE_INT32: + upb_handlers_setint32(h, key_field, putmapkey_int32_t, &empty_attr); + break; + case UPB_TYPE_INT64: + upb_handlers_setint64(h, key_field, putmapkey_int64_t, &empty_attr); + break; + case UPB_TYPE_UINT32: + upb_handlers_setuint32(h, key_field, putmapkey_uint32_t, &empty_attr); + break; + case UPB_TYPE_UINT64: + upb_handlers_setuint64(h, key_field, putmapkey_uint64_t, &empty_attr); + break; + case UPB_TYPE_BOOL: + upb_handlers_setbool(h, key_field, putmapkey_bool, &empty_attr); + break; + case UPB_TYPE_STRING: + upb_handlers_setstartstr(h, key_field, mapkeyval_startstr, &empty_attr); + upb_handlers_setstring(h, key_field, mapkey_str, &empty_attr); + upb_handlers_setendstr(h, key_field, mapkey_endstr, &empty_attr); + break; + case UPB_TYPE_BYTES: + upb_handlers_setstring(h, key_field, mapkey_bytes, &empty_attr); + break; + default: + assert(false); + break; + } + + switch (upb_fielddef_type(value_field)) { + case UPB_TYPE_INT32: + upb_handlers_setint32(h, value_field, putint32_t, &empty_attr); + break; + case UPB_TYPE_INT64: + upb_handlers_setint64(h, value_field, putint64_t, &empty_attr); + break; + case UPB_TYPE_UINT32: + upb_handlers_setuint32(h, value_field, putuint32_t, &empty_attr); + break; + case UPB_TYPE_UINT64: + upb_handlers_setuint64(h, value_field, putuint64_t, &empty_attr); + break; + case UPB_TYPE_BOOL: + upb_handlers_setbool(h, value_field, putbool, &empty_attr); + break; + case UPB_TYPE_FLOAT: + upb_handlers_setfloat(h, value_field, putfloat, &empty_attr); + break; + case UPB_TYPE_DOUBLE: + upb_handlers_setdouble(h, value_field, putdouble, &empty_attr); + break; + case UPB_TYPE_STRING: + upb_handlers_setstartstr(h, value_field, mapkeyval_startstr, &empty_attr); + upb_handlers_setstring(h, value_field, putstr, &empty_attr); + upb_handlers_setendstr(h, value_field, mapvalue_endstr, &empty_attr); + break; + case UPB_TYPE_BYTES: + upb_handlers_setstring(h, value_field, putbytes, &empty_attr); + break; + case UPB_TYPE_ENUM: { + upb_handlerattr enum_attr = UPB_HANDLERATTR_INITIALIZER; + set_enum_hd(h, value_field, &enum_attr); + upb_handlers_setint32(h, value_field, mapvalue_enum, &enum_attr); + upb_handlerattr_uninit(&enum_attr); + break; + } + case UPB_TYPE_MESSAGE: + // No handler necessary -- the submsg handlers will print the message + // as appropriate. + break; + } + + upb_handlerattr_uninit(&empty_attr); +} + +void printer_sethandlers(const void *closure, upb_handlers *h) { + UPB_UNUSED(closure); + const upb_msgdef *md = upb_handlers_msgdef(h); + bool is_mapentry = upb_msgdef_mapentry(md); upb_handlerattr empty_attr = UPB_HANDLERATTR_INITIALIZER; - upb_handlers_setstartmsg(h, startmap, &empty_attr); - upb_handlers_setendmsg(h, endmap, &empty_attr); - -#define TYPE(type, name, ctype) \ - case type: \ - if (upb_fielddef_isseq(f)) { \ - upb_handlers_set##name(h, f, repeated_##ctype, &empty_attr); \ - } else { \ - upb_handlers_set##name(h, f, scalar_##ctype, &name_attr); \ - } \ + + if (is_mapentry) { + // mapentry messages are sufficiently different that we handle them + // separately. + printer_sethandlers_mapentry(closure, h); + return; + } + + upb_handlers_setstartmsg(h, printer_startmsg, &empty_attr); + upb_handlers_setendmsg(h, printer_endmsg, &empty_attr); + +#define TYPE(type, name, ctype) \ + case type: \ + if (upb_fielddef_isseq(f)) { \ + upb_handlers_set##name(h, f, repeated_##ctype, &empty_attr); \ + } else { \ + upb_handlers_set##name(h, f, scalar_##ctype, &name_attr); \ + } \ break; - upb_msg_iter i; - upb_msg_begin(&i, upb_handlers_msgdef(h)); - for(; !upb_msg_done(&i); upb_msg_next(&i)) { + upb_msg_field_iter i; + upb_msg_field_begin(&i, md); + for(; !upb_msg_field_done(&i); upb_msg_field_next(&i)) { const upb_fielddef *f = upb_msg_iter_field(&i); upb_handlerattr name_attr = UPB_HANDLERATTR_INITIALIZER; upb_handlerattr_sethandlerdata(&name_attr, newstrpc(h, f)); - if (upb_fielddef_isseq(f)) { + if (upb_fielddef_ismap(f)) { + upb_handlers_setstartseq(h, f, startmap, &name_attr); + upb_handlers_setendseq(h, f, endmap, &name_attr); + } else if (upb_fielddef_isseq(f)) { upb_handlers_setstartseq(h, f, startseq, &name_attr); upb_handlers_setendseq(h, f, endseq, &empty_attr); } @@ -10352,12 +11141,8 @@ void printer_sethandlers(const void *closure, upb_handlers *h) { // For now, we always emit symbolic names for enums. We may want an // option later to control this behavior, but we will wait for a real // need first. - EnumHandlerData *hd = malloc(sizeof(EnumHandlerData)); - hd->enumdef = (const upb_enumdef *)upb_fielddef_subdef(f); - hd->keyname = newstrpc(h, f); - upb_handlers_addcleanup(h, hd, free); upb_handlerattr enum_attr = UPB_HANDLERATTR_INITIALIZER; - upb_handlerattr_sethandlerdata(&enum_attr, hd); + set_enum_hd(h, f, &enum_attr); if (upb_fielddef_isseq(f)) { upb_handlers_setint32(h, f, repeated_enum, &enum_attr); diff --git a/ruby/ext/google/protobuf_c/upb.h b/ruby/ext/google/protobuf_c/upb.h index fbcb8e99..8f6d3643 100644 --- a/ruby/ext/google/protobuf_c/upb.h +++ b/ruby/ext/google/protobuf_c/upb.h @@ -710,6 +710,9 @@ typedef struct { #define UPB_STRTABLE_INIT(count, mask, ctype, size_lg2, entries) \ {{count, mask, ctype, size_lg2, entries}} +#define UPB_EMPTY_STRTABLE_INIT(ctype) \ + UPB_STRTABLE_INIT(0, 0, ctype, 0, NULL) + typedef struct { upb_table t; // For entries that don't fit in the array part. const _upb_value *array; // Array part of the table. See const note above. @@ -1129,6 +1132,7 @@ class Def; class EnumDef; class FieldDef; class MessageDef; +class OneofDef; } #endif @@ -1136,6 +1140,7 @@ UPB_DECLARE_TYPE(upb::Def, upb_def); UPB_DECLARE_TYPE(upb::EnumDef, upb_enumdef); UPB_DECLARE_TYPE(upb::FieldDef, upb_fielddef); UPB_DECLARE_TYPE(upb::MessageDef, upb_msgdef); +UPB_DECLARE_TYPE(upb::OneofDef, upb_oneofdef); // Maximum field number allowed for FieldDefs. This is an inherent limit of the // protobuf wire format. @@ -1159,6 +1164,7 @@ typedef enum { UPB_DEF_MSG, UPB_DEF_FIELD, UPB_DEF_ENUM, + UPB_DEF_ONEOF, UPB_DEF_SERVICE, // Not yet implemented. UPB_DEF_ANY = -1, // Wildcard for upb_symtab_get*() } upb_deftype_t; @@ -1443,6 +1449,10 @@ UPB_DEFINE_DEF(upb::FieldDef, fielddef, FIELD, const MessageDef* containing_type() const; const char* containing_type_name(); + // The OneofDef to which this field belongs, or NULL if this field is not part + // of a oneof. + const OneofDef* containing_oneof() const; + // The field's type according to the enum in descriptor.proto. This is not // the same as UPB_TYPE_*, because it distinguishes between (for example) // INT32 and SINT32, whereas our "type" enum does not. This return of @@ -1456,6 +1466,7 @@ UPB_DEFINE_DEF(upb::FieldDef, fielddef, FIELD, bool IsString() const; bool IsSequence() const; bool IsPrimitive() const; + bool IsMap() const; // How integers are encoded. Only meaningful for integer types. // Defaults to UPB_INTFMT_VARIABLE, and is reset when "type" changes. @@ -1616,6 +1627,7 @@ UPB_DEFINE_STRUCT(upb_fielddef, upb_def, } sub; // The msgdef or enumdef for this field, if upb_hassubdef(f). bool subdef_is_symbolic; bool msg_is_symbolic; + const upb_oneofdef *oneof; bool default_is_string; bool type_is_set_; // False until type is explicitly set. bool is_extension_; @@ -1631,11 +1643,11 @@ UPB_DEFINE_STRUCT(upb_fielddef, upb_def, )); #define UPB_FIELDDEF_INIT(label, type, intfmt, tagdelim, is_extension, lazy, \ - packed, name, num, msgdef, subdef, selector_base, \ + packed, name, num, msgdef, subdef, selector_base, \ index, defaultval, refs, ref2s) \ { \ UPB_DEF_INIT(name, UPB_DEF_FIELD, refs, ref2s), defaultval, {msgdef}, \ - {subdef}, false, false, \ + {subdef}, NULL, false, false, \ type == UPB_TYPE_STRING || type == UPB_TYPE_BYTES, true, is_extension, \ lazy, packed, intfmt, tagdelim, type, label, num, selector_base, index \ } @@ -1669,6 +1681,7 @@ bool upb_fielddef_isextension(const upb_fielddef *f); bool upb_fielddef_lazy(const upb_fielddef *f); bool upb_fielddef_packed(const upb_fielddef *f); const upb_msgdef *upb_fielddef_containingtype(const upb_fielddef *f); +const upb_oneofdef *upb_fielddef_containingoneof(const upb_fielddef *f); upb_msgdef *upb_fielddef_containingtype_mutable(upb_fielddef *f); const char *upb_fielddef_containingtypename(upb_fielddef *f); upb_intfmt_t upb_fielddef_intfmt(const upb_fielddef *f); @@ -1678,6 +1691,7 @@ bool upb_fielddef_issubmsg(const upb_fielddef *f); bool upb_fielddef_isstring(const upb_fielddef *f); bool upb_fielddef_isseq(const upb_fielddef *f); bool upb_fielddef_isprimitive(const upb_fielddef *f); +bool upb_fielddef_ismap(const upb_fielddef *f); int64_t upb_fielddef_defaultint64(const upb_fielddef *f); int32_t upb_fielddef_defaultint32(const upb_fielddef *f); uint64_t upb_fielddef_defaultuint64(const upb_fielddef *f); @@ -1736,7 +1750,8 @@ UPB_END_EXTERN_C // } /* upb::MessageDef ************************************************************/ -typedef upb_inttable_iter upb_msg_iter; +typedef upb_inttable_iter upb_msg_field_iter; +typedef upb_strtable_iter upb_msg_oneof_iter; // Structure that describes a single .proto message type. // @@ -1766,14 +1781,37 @@ UPB_DEFINE_DEF(upb::MessageDef, msgdef, MSG, UPB_QUOTE( // The number of fields that belong to the MessageDef. int field_count() const; + // The number of oneofs that belong to the MessageDef. + int oneof_count() const; + // Adds a field (upb_fielddef object) to a msgdef. Requires that the msgdef // and the fielddefs are mutable. The fielddef's name and number must be // set, and the message may not already contain any field with this name or // number, and this fielddef may not be part of another message. In error // cases false is returned and the msgdef is unchanged. + // + // If the given field is part of a oneof, this call succeeds if and only if + // that oneof is already part of this msgdef. (Note that adding a oneof to a + // msgdef automatically adds all of its fields to the msgdef at the time that + // the oneof is added, so it is usually more idiomatic to add the oneof's + // fields first then add the oneof to the msgdef. This case is supported for + // convenience.) + // + // If |f| is already part of this MessageDef, this method performs no action + // and returns true (success). Thus, this method is idempotent. bool AddField(FieldDef* f, Status* s); bool AddField(const reffed_ptr<FieldDef>& f, Status* s); + // Adds a oneof (upb_oneofdef object) to a msgdef. Requires that the msgdef, + // oneof, and any fielddefs are mutable, that the fielddefs contained in the + // oneof do not have any name or number conflicts with existing fields in the + // msgdef, and that the oneof's name is unique among all oneofs in the msgdef. + // If the oneof is added successfully, all of its fields will be added + // directly to the msgdef as well. In error cases, false is returned and the + // msgdef is unchanged. + bool AddOneof(OneofDef* o, Status* s); + bool AddOneof(const reffed_ptr<OneofDef>& o, Status* s); + // These return NULL if the field is not found. FieldDef* FindFieldByNumber(uint32_t number); FieldDef* FindFieldByName(const char *name, size_t len); @@ -1797,6 +1835,25 @@ UPB_DEFINE_DEF(upb::MessageDef, msgdef, MSG, UPB_QUOTE( return FindFieldByName(str.c_str(), str.size()); } + OneofDef* FindOneofByName(const char* name, size_t len); + const OneofDef* FindOneofByName(const char* name, size_t len) const; + + OneofDef* FindOneofByName(const char* name) { + return FindOneofByName(name, strlen(name)); + } + const OneofDef* FindOneofByName(const char* name) const { + return FindOneofByName(name, strlen(name)); + } + + template<class T> + OneofDef* FindOneofByName(const T& str) { + return FindOneofByName(str.c_str(), str.size()); + } + template<class T> + const OneofDef* FindOneofByName(const T& str) const { + return FindOneofByName(str.c_str(), str.size()); + } + // Returns a new msgdef that is a copy of the given msgdef (and a copy of all // the fields) but with any references to submessages broken and replaced // with just the name of the submessage. Returns NULL if memory allocation @@ -1812,39 +1869,117 @@ UPB_DEFINE_DEF(upb::MessageDef, msgdef, MSG, UPB_QUOTE( bool mapentry() const; // Iteration over fields. The order is undefined. - class iterator : public std::iterator<std::forward_iterator_tag, FieldDef*> { + class field_iterator + : public std::iterator<std::forward_iterator_tag, FieldDef*> { public: - explicit iterator(MessageDef* md); - static iterator end(MessageDef* md); + explicit field_iterator(MessageDef* md); + static field_iterator end(MessageDef* md); void operator++(); FieldDef* operator*() const; - bool operator!=(const iterator& other) const; - bool operator==(const iterator& other) const; + bool operator!=(const field_iterator& other) const; + bool operator==(const field_iterator& other) const; private: - upb_msg_iter iter_; + upb_msg_field_iter iter_; }; - class const_iterator + class const_field_iterator : public std::iterator<std::forward_iterator_tag, const FieldDef*> { public: - explicit const_iterator(const MessageDef* md); - static const_iterator end(const MessageDef* md); + explicit const_field_iterator(const MessageDef* md); + static const_field_iterator end(const MessageDef* md); void operator++(); const FieldDef* operator*() const; - bool operator!=(const const_iterator& other) const; - bool operator==(const const_iterator& other) const; + bool operator!=(const const_field_iterator& other) const; + bool operator==(const const_field_iterator& other) const; private: - upb_msg_iter iter_; + upb_msg_field_iter iter_; }; - iterator begin(); - iterator end(); - const_iterator begin() const; - const_iterator end() const; + // Iteration over oneofs. The order is undefined. + class oneof_iterator + : public std::iterator<std::forward_iterator_tag, FieldDef*> { + public: + explicit oneof_iterator(MessageDef* md); + static oneof_iterator end(MessageDef* md); + + void operator++(); + OneofDef* operator*() const; + bool operator!=(const oneof_iterator& other) const; + bool operator==(const oneof_iterator& other) const; + + private: + upb_msg_oneof_iter iter_; + }; + + class const_oneof_iterator + : public std::iterator<std::forward_iterator_tag, const FieldDef*> { + public: + explicit const_oneof_iterator(const MessageDef* md); + static const_oneof_iterator end(const MessageDef* md); + + void operator++(); + const OneofDef* operator*() const; + bool operator!=(const const_oneof_iterator& other) const; + bool operator==(const const_oneof_iterator& other) const; + + private: + upb_msg_oneof_iter iter_; + }; + + class FieldAccessor { + public: + explicit FieldAccessor(MessageDef* msg) : msg_(msg) {} + field_iterator begin() { return msg_->field_begin(); } + field_iterator end() { return msg_->field_end(); } + private: + MessageDef* msg_; + }; + + class ConstFieldAccessor { + public: + explicit ConstFieldAccessor(const MessageDef* msg) : msg_(msg) {} + const_field_iterator begin() { return msg_->field_begin(); } + const_field_iterator end() { return msg_->field_end(); } + private: + const MessageDef* msg_; + }; + + class OneofAccessor { + public: + explicit OneofAccessor(MessageDef* msg) : msg_(msg) {} + oneof_iterator begin() { return msg_->oneof_begin(); } + oneof_iterator end() { return msg_->oneof_end(); } + private: + MessageDef* msg_; + }; + + class ConstOneofAccessor { + public: + explicit ConstOneofAccessor(const MessageDef* msg) : msg_(msg) {} + const_oneof_iterator begin() { return msg_->oneof_begin(); } + const_oneof_iterator end() { return msg_->oneof_end(); } + private: + const MessageDef* msg_; + }; + + field_iterator field_begin(); + field_iterator field_end(); + const_field_iterator field_begin() const; + const_field_iterator field_end() const; + + oneof_iterator oneof_begin(); + oneof_iterator oneof_end(); + const_oneof_iterator oneof_begin() const; + const_oneof_iterator oneof_end() const; + + FieldAccessor fields() { return FieldAccessor(this); } + ConstFieldAccessor fields() const { return ConstFieldAccessor(this); } + OneofAccessor oneofs() { return OneofAccessor(this); } + ConstOneofAccessor oneofs() const { return ConstOneofAccessor(this); } private: UPB_DISALLOW_POD_OPS(MessageDef, upb::MessageDef); @@ -1857,6 +1992,9 @@ UPB_DEFINE_STRUCT(upb_msgdef, upb_def, upb_inttable itof; // int to field upb_strtable ntof; // name to field + // Tables for looking up oneofs by name. + upb_strtable ntoo; // name to oneof + // Is this a map-entry message? // TODO: set this flag properly for static descriptors; regenerate // descriptor.upb.c. @@ -1865,11 +2003,14 @@ UPB_DEFINE_STRUCT(upb_msgdef, upb_def, // TODO(haberman): proper extension ranges (there can be multiple). )); +// TODO: also support static initialization of the oneofs table. This will be +// needed if we compile in descriptors that contain oneofs. #define UPB_MSGDEF_INIT(name, selector_count, submsg_field_count, itof, ntof, \ refs, ref2s) \ { \ UPB_DEF_INIT(name, UPB_DEF_MSG, refs, ref2s), selector_count, \ - submsg_field_count, itof, ntof, false \ + submsg_field_count, itof, ntof, \ + UPB_EMPTY_STRTABLE_INIT(UPB_CTYPE_PTR), false \ } UPB_BEGIN_EXTERN_C // { @@ -1893,6 +2034,8 @@ bool upb_msgdef_setfullname(upb_msgdef *m, const char *fullname, upb_status *s); upb_msgdef *upb_msgdef_dup(const upb_msgdef *m, const void *owner); bool upb_msgdef_addfield(upb_msgdef *m, upb_fielddef *f, const void *ref_donor, upb_status *s); +bool upb_msgdef_addoneof(upb_msgdef *m, upb_oneofdef *o, const void *ref_donor, + upb_status *s); // Field lookup in a couple of different variations: // - itof = int to field @@ -1917,11 +2060,38 @@ UPB_INLINE upb_fielddef *upb_msgdef_ntof_mutable(upb_msgdef *m, return (upb_fielddef *)upb_msgdef_ntof(m, name, len); } +// Oneof lookup: +// - ntoo = name to oneof +// - ntooz = name to oneof, null-terminated string. +const upb_oneofdef *upb_msgdef_ntoo(const upb_msgdef *m, const char *name, + size_t len); +int upb_msgdef_numoneofs(const upb_msgdef *m); + +UPB_INLINE const upb_oneofdef *upb_msgdef_ntooz(const upb_msgdef *m, + const char *name) { + return upb_msgdef_ntoo(m, name, strlen(name)); +} + +UPB_INLINE upb_oneofdef *upb_msgdef_ntoo_mutable(upb_msgdef *m, + const char *name, size_t len) { + return (upb_oneofdef *)upb_msgdef_ntoo(m, name, len); +} + void upb_msgdef_setmapentry(upb_msgdef *m, bool map_entry); bool upb_msgdef_mapentry(const upb_msgdef *m); -// upb_msg_iter i; -// for(upb_msg_begin(&i, m); !upb_msg_done(&i); upb_msg_next(&i)) { +// Well-known field tag numbers for map-entry messages. +#define UPB_MAPENTRY_KEY 1 +#define UPB_MAPENTRY_VALUE 2 + +const upb_oneofdef *upb_msgdef_findoneof(const upb_msgdef *m, + const char *name); +int upb_msgdef_numoneofs(const upb_msgdef *m); + +// upb_msg_field_iter i; +// for(upb_msg_field_begin(&i, m); +// !upb_msg_field_done(&i); +// upb_msg_field_next(&i)) { // upb_fielddef *f = upb_msg_iter_field(&i); // // ... // } @@ -1929,11 +2099,18 @@ bool upb_msgdef_mapentry(const upb_msgdef *m); // For C we don't have separate iterators for const and non-const. // It is the caller's responsibility to cast the upb_fielddef* to // const if the upb_msgdef* is const. -void upb_msg_begin(upb_msg_iter *iter, const upb_msgdef *m); -void upb_msg_next(upb_msg_iter *iter); -bool upb_msg_done(const upb_msg_iter *iter); -upb_fielddef *upb_msg_iter_field(const upb_msg_iter *iter); -void upb_msg_iter_setdone(upb_msg_iter *iter); +void upb_msg_field_begin(upb_msg_field_iter *iter, const upb_msgdef *m); +void upb_msg_field_next(upb_msg_field_iter *iter); +bool upb_msg_field_done(const upb_msg_field_iter *iter); +upb_fielddef *upb_msg_iter_field(const upb_msg_field_iter *iter); +void upb_msg_field_iter_setdone(upb_msg_field_iter *iter); + +// Similar to above, we also support iterating through the oneofs in a msgdef. +void upb_msg_oneof_begin(upb_msg_oneof_iter *iter, const upb_msgdef *m); +void upb_msg_oneof_next(upb_msg_oneof_iter *iter); +bool upb_msg_oneof_done(const upb_msg_oneof_iter *iter); +upb_oneofdef *upb_msg_iter_oneof(const upb_msg_oneof_iter *iter); +void upb_msg_oneof_iter_setdone(upb_msg_oneof_iter *iter); UPB_END_EXTERN_C // } @@ -2075,6 +2252,172 @@ int32_t upb_enum_iter_number(upb_enum_iter *iter); UPB_END_EXTERN_C // } +/* upb::OneofDef **************************************************************/ + +typedef upb_inttable_iter upb_oneof_iter; + +// Class that represents a oneof. Its base class is upb::Def (convert with +// upb::upcast()). +UPB_DEFINE_DEF(upb::OneofDef, oneofdef, ONEOF, UPB_QUOTE( + public: + // Returns NULL if memory allocation failed. + static reffed_ptr<OneofDef> New(); + + // Functionality from upb::RefCounted. + bool IsFrozen() const; + void Ref(const void* owner) const; + void Unref(const void* owner) const; + void DonateRef(const void* from, const void* to) const; + void CheckRef(const void* owner) const; + + // Functionality from upb::Def. + const char* full_name() const; + + // Returns the MessageDef that owns this OneofDef. + const MessageDef* containing_type() const; + + // Returns the name of this oneof. This is the name used to look up the oneof + // by name once added to a message def. + const char* name() const; + bool set_name(const char* name, Status* s); + + // Returns the number of fields currently defined in the oneof. + int field_count() const; + + // Adds a field to the oneof. The field must not have been added to any other + // oneof or msgdef. If the oneof is not yet part of a msgdef, then when the + // oneof is eventually added to a msgdef, all fields added to the oneof will + // also be added to the msgdef at that time. If the oneof is already part of a + // msgdef, the field must either be a part of that msgdef already, or must not + // be a part of any msgdef; in the latter case, the field is added to the + // msgdef as a part of this operation. + // + // The field may only have an OPTIONAL label, never REQUIRED or REPEATED. + // + // If |f| is already part of this MessageDef, this method performs no action + // and returns true (success). Thus, this method is idempotent. + bool AddField(FieldDef* field, Status* s); + bool AddField(const reffed_ptr<FieldDef>& field, Status* s); + + // Looks up by name. + const FieldDef* FindFieldByName(const char* name, size_t len) const; + FieldDef* FindFieldByName(const char* name, size_t len); + const FieldDef* FindFieldByName(const char* name) const { + return FindFieldByName(name, strlen(name)); + } + FieldDef* FindFieldByName(const char* name) { + return FindFieldByName(name, strlen(name)); + } + + template <class T> + FieldDef* FindFieldByName(const T& str) { + return FindFieldByName(str.c_str(), str.size()); + } + template <class T> + const FieldDef* FindFieldByName(const T& str) const { + return FindFieldByName(str.c_str(), str.size()); + } + + // Looks up by tag number. + const FieldDef* FindFieldByNumber(uint32_t num) const; + + // Returns a new OneofDef with all the same fields. The OneofDef will be owned + // by the given owner. + OneofDef* Dup(const void* owner) const; + + // Iteration over fields. The order is undefined. + class iterator : public std::iterator<std::forward_iterator_tag, FieldDef*> { + public: + explicit iterator(OneofDef* md); + static iterator end(OneofDef* md); + + void operator++(); + FieldDef* operator*() const; + bool operator!=(const iterator& other) const; + bool operator==(const iterator& other) const; + + private: + upb_oneof_iter iter_; + }; + + class const_iterator + : public std::iterator<std::forward_iterator_tag, const FieldDef*> { + public: + explicit const_iterator(const OneofDef* md); + static const_iterator end(const OneofDef* md); + + void operator++(); + const FieldDef* operator*() const; + bool operator!=(const const_iterator& other) const; + bool operator==(const const_iterator& other) const; + + private: + upb_oneof_iter iter_; + }; + + iterator begin(); + iterator end(); + const_iterator begin() const; + const_iterator end() const; + + private: + UPB_DISALLOW_POD_OPS(OneofDef, upb::OneofDef); +), +UPB_DEFINE_STRUCT(upb_oneofdef, upb_def, + upb_strtable ntof; + upb_inttable itof; + const upb_msgdef *parent; +)); + +#define UPB_ONEOFDEF_INIT(name, ntof, itof, refs, ref2s) \ + { UPB_DEF_INIT(name, UPB_DEF_ENUM, refs, ref2s), ntof, itof } + +UPB_BEGIN_EXTERN_C // { + +// Native C API. +upb_oneofdef *upb_oneofdef_new(const void *owner); +upb_oneofdef *upb_oneofdef_dup(const upb_oneofdef *o, const void *owner); + +// From upb_refcounted. +void upb_oneofdef_unref(const upb_oneofdef *o, const void *owner); +bool upb_oneofdef_isfrozen(const upb_oneofdef *e); +void upb_oneofdef_ref(const upb_oneofdef *o, const void *owner); +void upb_oneofdef_donateref(const upb_oneofdef *m, const void *from, + const void *to); +void upb_oneofdef_checkref(const upb_oneofdef *o, const void *owner); + +const char *upb_oneofdef_name(const upb_oneofdef *o); +bool upb_oneofdef_setname(upb_oneofdef *o, const char *name, upb_status *s); + +const upb_msgdef *upb_oneofdef_containingtype(const upb_oneofdef *o); +int upb_oneofdef_numfields(const upb_oneofdef *o); +bool upb_oneofdef_addfield(upb_oneofdef *o, upb_fielddef *f, + const void *ref_donor, + upb_status *s); + +// Oneof lookups: +// - ntof: look up a field by name. +// - ntofz: look up a field by name (as a null-terminated string). +// - itof: look up a field by number. +const upb_fielddef *upb_oneofdef_ntof(const upb_oneofdef *o, + const char *name, size_t length); +UPB_INLINE const upb_fielddef *upb_oneofdef_ntofz(const upb_oneofdef *o, + const char *name) { + return upb_oneofdef_ntof(o, name, strlen(name)); +} +const upb_fielddef *upb_oneofdef_itof(const upb_oneofdef *o, uint32_t num); + +// upb_oneof_iter i; +// for(upb_oneof_begin(&i, e); !upb_oneof_done(&i); upb_oneof_next(&i)) { +// // ... +// } +void upb_oneof_begin(upb_oneof_iter *iter, const upb_oneofdef *o); +void upb_oneof_next(upb_oneof_iter *iter); +bool upb_oneof_done(upb_oneof_iter *iter); +upb_fielddef *upb_oneof_iter_field(const upb_oneof_iter *iter); +void upb_oneof_iter_setdone(upb_oneof_iter *iter); + +UPB_END_EXTERN_C // } #ifdef __cplusplus @@ -2201,6 +2544,9 @@ inline void FieldDef::set_packed(bool packed) { inline const MessageDef* FieldDef::containing_type() const { return upb_fielddef_containingtype(this); } +inline const OneofDef* FieldDef::containing_oneof() const { + return upb_fielddef_containingoneof(this); +} inline const char* FieldDef::containing_type_name() { return upb_fielddef_containingtypename(this); } @@ -2237,6 +2583,7 @@ inline bool FieldDef::IsSubMessage() const { } inline bool FieldDef::IsString() const { return upb_fielddef_isstring(this); } inline bool FieldDef::IsSequence() const { return upb_fielddef_isseq(this); } +inline bool FieldDef::IsMap() const { return upb_fielddef_ismap(this); } inline int64_t FieldDef::default_int64() const { return upb_fielddef_defaultint64(this); } @@ -2351,12 +2698,21 @@ inline bool MessageDef::Freeze(Status* status) { inline int MessageDef::field_count() const { return upb_msgdef_numfields(this); } +inline int MessageDef::oneof_count() const { + return upb_msgdef_numoneofs(this); +} inline bool MessageDef::AddField(upb_fielddef* f, Status* s) { return upb_msgdef_addfield(this, f, NULL, s); } inline bool MessageDef::AddField(const reffed_ptr<FieldDef>& f, Status* s) { return upb_msgdef_addfield(this, f.get(), NULL, s); } +inline bool MessageDef::AddOneof(upb_oneofdef* o, Status* s) { + return upb_msgdef_addoneof(this, o, NULL, s); +} +inline bool MessageDef::AddOneof(const reffed_ptr<OneofDef>& o, Status* s) { + return upb_msgdef_addoneof(this, o.get(), NULL, s); +} inline FieldDef* MessageDef::FindFieldByNumber(uint32_t number) { return upb_msgdef_itof_mutable(this, number); } @@ -2370,6 +2726,13 @@ inline const FieldDef *MessageDef::FindFieldByName(const char *name, size_t len) const { return upb_msgdef_ntof(this, name, len); } +inline OneofDef* MessageDef::FindOneofByName(const char* name, size_t len) { + return upb_msgdef_ntoo_mutable(this, name, len); +} +inline const OneofDef* MessageDef::FindOneofByName(const char* name, + size_t len) const { + return upb_msgdef_ntoo(this, name, len); +} inline MessageDef* MessageDef::Dup(const void *owner) const { return upb_msgdef_dup(this, owner); } @@ -2379,55 +2742,127 @@ inline void MessageDef::setmapentry(bool map_entry) { inline bool MessageDef::mapentry() const { return upb_msgdef_mapentry(this); } -inline MessageDef::iterator MessageDef::begin() { return iterator(this); } -inline MessageDef::iterator MessageDef::end() { return iterator::end(this); } -inline MessageDef::const_iterator MessageDef::begin() const { - return const_iterator(this); +inline MessageDef::field_iterator MessageDef::field_begin() { + return field_iterator(this); } -inline MessageDef::const_iterator MessageDef::end() const { - return const_iterator::end(this); +inline MessageDef::field_iterator MessageDef::field_end() { + return field_iterator::end(this); +} +inline MessageDef::const_field_iterator MessageDef::field_begin() const { + return const_field_iterator(this); +} +inline MessageDef::const_field_iterator MessageDef::field_end() const { + return const_field_iterator::end(this); } -inline MessageDef::iterator::iterator(MessageDef* md) { - upb_msg_begin(&iter_, md); +inline MessageDef::oneof_iterator MessageDef::oneof_begin() { + return oneof_iterator(this); +} +inline MessageDef::oneof_iterator MessageDef::oneof_end() { + return oneof_iterator::end(this); +} +inline MessageDef::const_oneof_iterator MessageDef::oneof_begin() const { + return const_oneof_iterator(this); +} +inline MessageDef::const_oneof_iterator MessageDef::oneof_end() const { + return const_oneof_iterator::end(this); } -inline MessageDef::iterator MessageDef::iterator::end(MessageDef* md) { - MessageDef::iterator iter(md); - upb_msg_iter_setdone(&iter.iter_); + +inline MessageDef::field_iterator::field_iterator(MessageDef* md) { + upb_msg_field_begin(&iter_, md); +} +inline MessageDef::field_iterator MessageDef::field_iterator::end( + MessageDef* md) { + MessageDef::field_iterator iter(md); + upb_msg_field_iter_setdone(&iter.iter_); return iter; } -inline FieldDef* MessageDef::iterator::operator*() const { +inline FieldDef* MessageDef::field_iterator::operator*() const { return upb_msg_iter_field(&iter_); } -inline void MessageDef::iterator::operator++() { return upb_msg_next(&iter_); } -inline bool MessageDef::iterator::operator==(const iterator &other) const { +inline void MessageDef::field_iterator::operator++() { + return upb_msg_field_next(&iter_); +} +inline bool MessageDef::field_iterator::operator==( + const field_iterator &other) const { return upb_inttable_iter_isequal(&iter_, &other.iter_); } -inline bool MessageDef::iterator::operator!=(const iterator &other) const { +inline bool MessageDef::field_iterator::operator!=( + const field_iterator &other) const { return !(*this == other); } -inline MessageDef::const_iterator::const_iterator(const MessageDef* md) { - upb_msg_begin(&iter_, md); +inline MessageDef::const_field_iterator::const_field_iterator( + const MessageDef* md) { + upb_msg_field_begin(&iter_, md); } -inline MessageDef::const_iterator MessageDef::const_iterator::end( +inline MessageDef::const_field_iterator MessageDef::const_field_iterator::end( const MessageDef *md) { - MessageDef::const_iterator iter(md); - upb_msg_iter_setdone(&iter.iter_); + MessageDef::const_field_iterator iter(md); + upb_msg_field_iter_setdone(&iter.iter_); return iter; } -inline const FieldDef* MessageDef::const_iterator::operator*() const { +inline const FieldDef* MessageDef::const_field_iterator::operator*() const { return upb_msg_iter_field(&iter_); } -inline void MessageDef::const_iterator::operator++() { - return upb_msg_next(&iter_); +inline void MessageDef::const_field_iterator::operator++() { + return upb_msg_field_next(&iter_); } -inline bool MessageDef::const_iterator::operator==( - const const_iterator &other) const { +inline bool MessageDef::const_field_iterator::operator==( + const const_field_iterator &other) const { return upb_inttable_iter_isequal(&iter_, &other.iter_); } -inline bool MessageDef::const_iterator::operator!=( - const const_iterator &other) const { +inline bool MessageDef::const_field_iterator::operator!=( + const const_field_iterator &other) const { + return !(*this == other); +} + +inline MessageDef::oneof_iterator::oneof_iterator(MessageDef* md) { + upb_msg_oneof_begin(&iter_, md); +} +inline MessageDef::oneof_iterator MessageDef::oneof_iterator::end( + MessageDef* md) { + MessageDef::oneof_iterator iter(md); + upb_msg_oneof_iter_setdone(&iter.iter_); + return iter; +} +inline OneofDef* MessageDef::oneof_iterator::operator*() const { + return upb_msg_iter_oneof(&iter_); +} +inline void MessageDef::oneof_iterator::operator++() { + return upb_msg_oneof_next(&iter_); +} +inline bool MessageDef::oneof_iterator::operator==( + const oneof_iterator &other) const { + return upb_strtable_iter_isequal(&iter_, &other.iter_); +} +inline bool MessageDef::oneof_iterator::operator!=( + const oneof_iterator &other) const { + return !(*this == other); +} + +inline MessageDef::const_oneof_iterator::const_oneof_iterator( + const MessageDef* md) { + upb_msg_oneof_begin(&iter_, md); +} +inline MessageDef::const_oneof_iterator MessageDef::const_oneof_iterator::end( + const MessageDef *md) { + MessageDef::const_oneof_iterator iter(md); + upb_msg_oneof_iter_setdone(&iter.iter_); + return iter; +} +inline const OneofDef* MessageDef::const_oneof_iterator::operator*() const { + return upb_msg_iter_oneof(&iter_); +} +inline void MessageDef::const_oneof_iterator::operator++() { + return upb_msg_oneof_next(&iter_); +} +inline bool MessageDef::const_oneof_iterator::operator==( + const const_oneof_iterator &other) const { + return upb_strtable_iter_isequal(&iter_, &other.iter_); +} +inline bool MessageDef::const_oneof_iterator::operator!=( + const const_oneof_iterator &other) const { return !(*this == other); } @@ -2495,6 +2930,105 @@ inline const char* EnumDef::Iterator::name() { } inline bool EnumDef::Iterator::Done() { return upb_enum_done(&iter_); } inline void EnumDef::Iterator::Next() { return upb_enum_next(&iter_); } + +inline reffed_ptr<OneofDef> OneofDef::New() { + upb_oneofdef *o = upb_oneofdef_new(&o); + return reffed_ptr<OneofDef>(o, &o); +} +inline bool OneofDef::IsFrozen() const { return upb_oneofdef_isfrozen(this); } +inline void OneofDef::Ref(const void* owner) const { + return upb_oneofdef_ref(this, owner); +} +inline void OneofDef::Unref(const void* owner) const { + return upb_oneofdef_unref(this, owner); +} +inline void OneofDef::DonateRef(const void* from, const void* to) const { + return upb_oneofdef_donateref(this, from, to); +} +inline void OneofDef::CheckRef(const void* owner) const { + return upb_oneofdef_checkref(this, owner); +} +inline const char* OneofDef::full_name() const { + return upb_oneofdef_name(this); +} + +inline const MessageDef* OneofDef::containing_type() const { + return upb_oneofdef_containingtype(this); +} +inline const char* OneofDef::name() const { + return upb_oneofdef_name(this); +} +inline bool OneofDef::set_name(const char* name, Status* s) { + return upb_oneofdef_setname(this, name, s); +} +inline int OneofDef::field_count() const { + return upb_oneofdef_numfields(this); +} +inline bool OneofDef::AddField(FieldDef* field, Status* s) { + return upb_oneofdef_addfield(this, field, NULL, s); +} +inline bool OneofDef::AddField(const reffed_ptr<FieldDef>& field, Status* s) { + return upb_oneofdef_addfield(this, field.get(), NULL, s); +} +inline const FieldDef* OneofDef::FindFieldByName(const char* name, + size_t len) const { + return upb_oneofdef_ntof(this, name, len); +} +inline const FieldDef* OneofDef::FindFieldByNumber(uint32_t num) const { + return upb_oneofdef_itof(this, num); +} +inline OneofDef::iterator OneofDef::begin() { return iterator(this); } +inline OneofDef::iterator OneofDef::end() { return iterator::end(this); } +inline OneofDef::const_iterator OneofDef::begin() const { + return const_iterator(this); +} +inline OneofDef::const_iterator OneofDef::end() const { + return const_iterator::end(this); +} + +inline OneofDef::iterator::iterator(OneofDef* o) { + upb_oneof_begin(&iter_, o); +} +inline OneofDef::iterator OneofDef::iterator::end(OneofDef* o) { + OneofDef::iterator iter(o); + upb_oneof_iter_setdone(&iter.iter_); + return iter; +} +inline FieldDef* OneofDef::iterator::operator*() const { + return upb_oneof_iter_field(&iter_); +} +inline void OneofDef::iterator::operator++() { return upb_oneof_next(&iter_); } +inline bool OneofDef::iterator::operator==(const iterator &other) const { + return upb_inttable_iter_isequal(&iter_, &other.iter_); +} +inline bool OneofDef::iterator::operator!=(const iterator &other) const { + return !(*this == other); +} + +inline OneofDef::const_iterator::const_iterator(const OneofDef* md) { + upb_oneof_begin(&iter_, md); +} +inline OneofDef::const_iterator OneofDef::const_iterator::end( + const OneofDef *md) { + OneofDef::const_iterator iter(md); + upb_oneof_iter_setdone(&iter.iter_); + return iter; +} +inline const FieldDef* OneofDef::const_iterator::operator*() const { + return upb_msg_iter_field(&iter_); +} +inline void OneofDef::const_iterator::operator++() { + return upb_oneof_next(&iter_); +} +inline bool OneofDef::const_iterator::operator==( + const const_iterator &other) const { + return upb_inttable_iter_isequal(&iter_, &other.iter_); +} +inline bool OneofDef::const_iterator::operator!=( + const const_iterator &other) const { + return !(*this == other); +} + } // namespace upb #endif @@ -7295,12 +7829,30 @@ class Parser; UPB_DECLARE_TYPE(upb::json::Parser, upb_json_parser); -// Internal-only struct used by the parser. +// Internal-only struct used by the parser. A parser frame corresponds +// one-to-one with a handler (sink) frame. typedef struct { UPB_PRIVATE_FOR_CPP upb_sink sink; + // The current message in which we're parsing, and the field whose value we're + // expecting next. const upb_msgdef *m; const upb_fielddef *f; + + // We are in a repeated-field context, ready to emit mapentries as + // submessages. This flag alters the start-of-object (open-brace) behavior to + // begin a sequence of mapentry messages rather than a single submessage. + bool is_map; + // We are in a map-entry message context. This flag is set when parsing the + // value field of a single map entry and indicates to all value-field parsers + // (subobjects, strings, numbers, and bools) that the map-entry submessage + // should end as soon as the value is parsed. + bool is_mapentry; + // If |is_map| or |is_mapentry| is true, |mapfield| refers to the parent + // message's map field that we're currently parsing. This differs from |f| + // because |f| is the field in the *current* message (i.e., the map-entry + // message itself), not the parent's field that leads to this map. + const upb_fielddef *mapfield; } upb_jsonparser_frame; diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index 87033ac4..e294751f 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -7,7 +7,7 @@ end Gem::Specification.new do |s| s.name = "google-protobuf" - s.version = "3.0.0.alpha.2" + s.version = "3.0.0.alpha.2.0" s.licenses = ["BSD"] s.summary = "Protocol Buffers" s.description = "Protocol Buffers are Google's data interchange format." @@ -18,5 +18,7 @@ Gem::Specification.new do |s| s.files = ["lib/google/protobuf.rb"] + # extension C source find_c_source("ext/google/protobuf_c") - s.test_files = `git ls-files -- tests`.split + s.test_files = ["tests/basic.rb", + "tests/stress.rb", + "tests/generated_code_test.rb"] end diff --git a/ruby/tests/basic.rb b/ruby/tests/basic.rb index 92655033..a78cc394 100644 --- a/ruby/tests/basic.rb +++ b/ruby/tests/basic.rb @@ -73,6 +73,15 @@ module BasicTest optional :key, :string, 1 optional :value, :message, 2, "TestMessage2" end + + add_message "OneofMessage" do + oneof :my_oneof do + optional :a, :string, 1 + optional :b, :int32, 2 + optional :c, :message, 3, "TestMessage2" + optional :d, :enum, 4, "TestEnum" + end + end end TestMessage = pool.lookup("TestMessage").msgclass @@ -87,6 +96,7 @@ module BasicTest pool.lookup("MapMessageWireEquiv_entry1").msgclass MapMessageWireEquiv_entry2 = pool.lookup("MapMessageWireEquiv_entry2").msgclass + OneofMessage = pool.lookup("OneofMessage").msgclass # ------------ test cases --------------- @@ -226,7 +236,8 @@ module BasicTest assert l.count == 0 l = Google::Protobuf::RepeatedField.new(:int32, [1, 2, 3]) assert l.count == 3 - assert l == [1, 2, 3] + assert_equal [1, 2, 3], l + assert_equal l, [1, 2, 3] l.push 4 assert l == [1, 2, 3, 4] dst_list = [] @@ -380,7 +391,7 @@ module BasicTest # We only assert on inspect value when there is one map entry because the # order in which elements appear is unspecified (depends on the internal # hash function). We don't want a brittle test. - assert m.inspect == "{\"jkl;\" => 42}" + assert m.inspect == "{\"jkl;\"=>42}" assert m.keys == ["jkl;"] assert m.values == [42] @@ -582,6 +593,91 @@ module BasicTest "b" => TestMessage2.new(:foo => 2)} end + def test_oneof_descriptors + d = OneofMessage.descriptor + o = d.lookup_oneof("my_oneof") + assert o != nil + assert o.class == Google::Protobuf::OneofDescriptor + assert o.name == "my_oneof" + oneof_count = 0 + d.each_oneof{ |oneof| + oneof_count += 1 + assert oneof == o + } + assert oneof_count == 1 + assert o.count == 4 + field_names = o.map{|f| f.name}.sort + assert field_names == ["a", "b", "c", "d"] + end + + def test_oneof + d = OneofMessage.new + assert d.a == nil + assert d.b == nil + assert d.c == nil + assert d.d == nil + assert d.my_oneof == nil + + d.a = "hi" + assert d.a == "hi" + assert d.b == nil + assert d.c == nil + assert d.d == nil + assert d.my_oneof == :a + + d.b = 42 + assert d.a == nil + assert d.b == 42 + assert d.c == nil + assert d.d == nil + assert d.my_oneof == :b + + d.c = TestMessage2.new(:foo => 100) + assert d.a == nil + assert d.b == nil + assert d.c.foo == 100 + assert d.d == nil + assert d.my_oneof == :c + + d.d = :C + assert d.a == nil + assert d.b == nil + assert d.c == nil + assert d.d == :C + assert d.my_oneof == :d + + d2 = OneofMessage.decode(OneofMessage.encode(d)) + assert d2 == d + + encoded_field_a = OneofMessage.encode(OneofMessage.new(:a => "string")) + encoded_field_b = OneofMessage.encode(OneofMessage.new(:b => 1000)) + encoded_field_c = OneofMessage.encode( + OneofMessage.new(:c => TestMessage2.new(:foo => 1))) + encoded_field_d = OneofMessage.encode(OneofMessage.new(:d => :B)) + + d3 = OneofMessage.decode( + encoded_field_c + encoded_field_a + encoded_field_d) + assert d3.a == nil + assert d3.b == nil + assert d3.c == nil + assert d3.d == :B + + d4 = OneofMessage.decode( + encoded_field_c + encoded_field_a + encoded_field_d + + encoded_field_c) + assert d4.a == nil + assert d4.b == nil + assert d4.c.foo == 1 + assert d4.d == nil + + d5 = OneofMessage.new(:a => "hello") + assert d5.a != nil + d5.a = nil + assert d5.a == nil + assert OneofMessage.encode(d5) == '' + assert d5.my_oneof == nil + end + def test_enum_field m = TestMessage.new assert m.optional_enum == :Default @@ -621,6 +717,14 @@ module BasicTest assert m.repeated_msg[0].object_id != m2.repeated_msg[0].object_id end + def test_eq + m = TestMessage.new(:optional_int32 => 42, + :repeated_int32 => [1, 2, 3]) + m2 = TestMessage.new(:optional_int32 => 43, + :repeated_int32 => [1, 2, 3]) + assert m != m2 + end + def test_enum_lookup assert TestEnum::A == 1 assert TestEnum::B == 2 @@ -888,5 +992,13 @@ module BasicTest m2 = TestMessage.decode_json(json_text) assert m == m2 end + + def test_json_maps + m = MapMessage.new(:map_string_int32 => {"a" => 1}) + expected = '{"map_string_int32":{"a":1},"map_string_msg":{}}' + assert MapMessage.encode_json(m) == expected + m2 = MapMessage.decode_json(MapMessage.encode_json(m)) + assert m == m2 + end end end diff --git a/ruby/tests/generated_code.proto b/ruby/tests/generated_code.proto new file mode 100644 index 00000000..b1d63232 --- /dev/null +++ b/ruby/tests/generated_code.proto @@ -0,0 +1,67 @@ +syntax = "proto3"; + +package A.B.C; + +message TestMessage { + optional int32 optional_int32 = 1; + optional int64 optional_int64 = 2; + optional uint32 optional_uint32 = 3; + optional uint64 optional_uint64 = 4; + optional bool optional_bool = 5; + optional double optional_double = 6; + optional float optional_float = 7; + optional string optional_string = 8; + optional bytes optional_bytes = 9; + optional TestEnum optional_enum = 10; + optional TestMessage optional_msg = 11; + + repeated int32 repeated_int32 = 21; + repeated int64 repeated_int64 = 22; + repeated uint32 repeated_uint32 = 23; + repeated uint64 repeated_uint64 = 24; + repeated bool repeated_bool = 25; + repeated double repeated_double = 26; + repeated float repeated_float = 27; + repeated string repeated_string = 28; + repeated bytes repeated_bytes = 29; + repeated TestEnum repeated_enum = 30; + repeated TestMessage repeated_msg = 31; + + oneof my_oneof { + int32 oneof_int32 = 41; + int64 oneof_int64 = 42; + uint32 oneof_uint32 = 43; + uint64 oneof_uint64 = 44; + bool oneof_bool = 45; + double oneof_double = 46; + float oneof_float = 47; + string oneof_string = 48; + bytes oneof_bytes = 49; + TestEnum oneof_enum = 50; + TestMessage oneof_msg = 51; + } + + map<int32, string> map_int32_string = 61; + map<int64, string> map_int64_string = 62; + map<uint32, string> map_uint32_string = 63; + map<uint64, string> map_uint64_string = 64; + map<bool, string> map_bool_string = 65; + map<string, string> map_string_string = 66; + map<string, TestMessage> map_string_msg = 67; + map<string, TestEnum> map_string_enum = 68; + map<string, int32> map_string_int32 = 69; + map<string, bool> map_string_bool = 70; + + message NestedMessage { + optional int32 foo = 1; + } + + optional NestedMessage nested_message = 80; +} + +enum TestEnum { + Default = 0; + A = 1; + B = 2; + C = 3; +} diff --git a/ruby/tests/generated_code.rb b/ruby/tests/generated_code.rb new file mode 100644 index 00000000..5a685433 --- /dev/null +++ b/ruby/tests/generated_code.rb @@ -0,0 +1,74 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: generated_code.proto + +require 'google/protobuf' + +Google::Protobuf::DescriptorPool.generated_pool.build do + add_message "A.B.C.TestMessage" do + optional :optional_int32, :int32, 1 + optional :optional_int64, :int64, 2 + optional :optional_uint32, :uint32, 3 + optional :optional_uint64, :uint64, 4 + optional :optional_bool, :bool, 5 + optional :optional_double, :double, 6 + optional :optional_float, :float, 7 + optional :optional_string, :string, 8 + optional :optional_bytes, :string, 9 + optional :optional_enum, :enum, 10, "A.B.C.TestEnum" + optional :optional_msg, :message, 11, "A.B.C.TestMessage" + repeated :repeated_int32, :int32, 21 + repeated :repeated_int64, :int64, 22 + repeated :repeated_uint32, :uint32, 23 + repeated :repeated_uint64, :uint64, 24 + repeated :repeated_bool, :bool, 25 + repeated :repeated_double, :double, 26 + repeated :repeated_float, :float, 27 + repeated :repeated_string, :string, 28 + repeated :repeated_bytes, :string, 29 + repeated :repeated_enum, :enum, 30, "A.B.C.TestEnum" + repeated :repeated_msg, :message, 31, "A.B.C.TestMessage" + map :map_int32_string, :int32, :string, 61 + map :map_int64_string, :int64, :string, 62 + map :map_uint32_string, :uint32, :string, 63 + map :map_uint64_string, :uint64, :string, 64 + map :map_bool_string, :bool, :string, 65 + map :map_string_string, :string, :string, 66 + map :map_string_msg, :string, :message, 67, "A.B.C.TestMessage" + map :map_string_enum, :string, :enum, 68, "A.B.C.TestEnum" + map :map_string_int32, :string, :int32, 69 + map :map_string_bool, :string, :bool, 70 + optional :nested_message, :message, 80, "A.B.C.TestMessage.NestedMessage" + oneof :my_oneof do + optional :oneof_int32, :int32, 41 + optional :oneof_int64, :int64, 42 + optional :oneof_uint32, :uint32, 43 + optional :oneof_uint64, :uint64, 44 + optional :oneof_bool, :bool, 45 + optional :oneof_double, :double, 46 + optional :oneof_float, :float, 47 + optional :oneof_string, :string, 48 + optional :oneof_bytes, :string, 49 + optional :oneof_enum, :enum, 50, "A.B.C.TestEnum" + optional :oneof_msg, :message, 51, "A.B.C.TestMessage" + end + end + add_message "A.B.C.TestMessage.NestedMessage" do + optional :foo, :int32, 1 + end + add_enum "A.B.C.TestEnum" do + value :Default, 0 + value :A, 1 + value :B, 2 + value :C, 3 + end +end + +module A + module B + module C + TestMessage = Google::Protobuf::DescriptorPool.generated_pool.lookup("A.B.C.TestMessage").msgclass + TestMessage::NestedMessage = Google::Protobuf::DescriptorPool.generated_pool.lookup("A.B.C.TestMessage.NestedMessage").msgclass + TestEnum = Google::Protobuf::DescriptorPool.generated_pool.lookup("A.B.C.TestEnum").enummodule + end + end +end diff --git a/ruby/tests/generated_code_test.rb b/ruby/tests/generated_code_test.rb new file mode 100644 index 00000000..daef357a --- /dev/null +++ b/ruby/tests/generated_code_test.rb @@ -0,0 +1,17 @@ +#!/usr/bin/ruby + +# generated_code.rb is in the same directory as this test. +$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__))) + +require 'generated_code' +require 'test/unit' + +class GeneratedCodeTest < Test::Unit::TestCase + def test_generated_msg + # just test that we can instantiate the message. The purpose of this test + # is to ensure that the output of the code generator is valid Ruby and + # successfully creates message definitions and classes, not to test every + # aspect of the extension (basic.rb is for that). + m = A::B::C::TestMessage.new() + end +end diff --git a/src/Makefile.am b/src/Makefile.am index 3a469fd7..14756ff8 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -31,12 +31,24 @@ nobase_dist_proto_DATA = google/protobuf/descriptor.proto \ clean-local: rm -f *.loT -CLEANFILES = $(protoc_outputs) unittest_proto_middleman \ +public_config = google/protobuf/stubs/pbconfig.h + +CLEANFILES = $(protoc_outputs) $(public_config) unittest_proto_middleman \ testzip.jar testzip.list testzip.proto testzip.zip MAINTAINERCLEANFILES = \ Makefile.in +# Generate and distribute a minimum config.h file to make hash_map work. +# The autoheader config has too much info, which might conflict with other +# macros applications might include. Thus, we create a stubs/pbconfig.h, that +# only #defines what we really need, and prefix it with GOOGLE_PROTOBUF_ to +# avoid conflicts. +$(public_config): $(top_builddir)/config.h $(top_srcdir)/config.h.include + echo "// Note: Google Protobuf internal only. Do NOT include." > $@ + cat $(top_builddir)/config.h | grep -f $(top_srcdir)/config.h.include | \ + sed 's,#define , #define GOOGLE_PROTOBUF_,' >> $@ + nobase_include_HEADERS = \ google/protobuf/stubs/atomic_sequence_num.h \ google/protobuf/stubs/atomicops.h \ @@ -54,11 +66,13 @@ nobase_include_HEADERS = \ google/protobuf/stubs/atomicops_internals_x86_gcc.h \ google/protobuf/stubs/atomicops_internals_x86_msvc.h \ google/protobuf/stubs/casts.h \ - google/protobuf/stubs/singleton.h \ google/protobuf/stubs/common.h \ google/protobuf/stubs/fastmem.h \ + google/protobuf/stubs/hash.h \ google/protobuf/stubs/once.h \ google/protobuf/stubs/platform_macros.h \ + google/protobuf/stubs/shared_ptr.h \ + google/protobuf/stubs/singleton.h \ google/protobuf/stubs/stl_util.h \ google/protobuf/stubs/template_util.h \ google/protobuf/stubs/type_traits.h \ @@ -111,6 +125,9 @@ nobase_include_HEADERS = \ google/protobuf/compiler/python/python_generator.h \ google/protobuf/compiler/ruby/ruby_generator.h +nobase_nodist_include_HEADERS = \ + $(public_config) + lib_LTLIBRARIES = libprotobuf-lite.la libprotobuf.la libprotoc.la libprotobuf_lite_la_LIBADD = $(PTHREAD_LIBS) @@ -136,6 +153,7 @@ libprotobuf_lite_la_SOURCES = \ google/protobuf/io/coded_stream_inl.h \ google/protobuf/io/zero_copy_stream.cc \ google/protobuf/io/zero_copy_stream_impl_lite.cc +nodist_libprotobuf_lite_la_SOURCES = $(public_config) libprotobuf_la_LIBADD = $(PTHREAD_LIBS) libprotobuf_la_LDFLAGS = -version-info 10:0:0 -export-dynamic -no-undefined @@ -167,6 +185,7 @@ libprotobuf_la_SOURCES = \ google/protobuf/io/zero_copy_stream_impl.cc \ google/protobuf/compiler/importer.cc \ google/protobuf/compiler/parser.cc +nodist_libprotobuf_la_SOURCES = $(nodist_libprotobuf_lite_la_SOURCES) libprotoc_la_LIBADD = $(PTHREAD_LIBS) libprotobuf.la libprotoc_la_LDFLAGS = -version-info 10:0:0 -export-dynamic -no-undefined @@ -243,26 +262,28 @@ libprotoc_la_SOURCES = \ google/protobuf/compiler/java/java_doc_comment.cc \ google/protobuf/compiler/java/java_doc_comment.h \ google/protobuf/compiler/javanano/javanano_enum.cc \ + google/protobuf/compiler/javanano/javanano_enum.h \ + google/protobuf/compiler/javanano/javanano_enum_field.cc \ google/protobuf/compiler/javanano/javanano_enum_field.h \ google/protobuf/compiler/javanano/javanano_extension.cc \ + google/protobuf/compiler/javanano/javanano_extension.h \ google/protobuf/compiler/javanano/javanano_field.cc \ + google/protobuf/compiler/javanano/javanano_field.h \ google/protobuf/compiler/javanano/javanano_file.cc \ + google/protobuf/compiler/javanano/javanano_file.h \ google/protobuf/compiler/javanano/javanano_generator.cc \ + google/protobuf/compiler/javanano/javanano_generator.h \ google/protobuf/compiler/javanano/javanano_helpers.cc \ + google/protobuf/compiler/javanano/javanano_helpers.h \ + google/protobuf/compiler/javanano/javanano_map_field.cc \ + google/protobuf/compiler/javanano/javanano_map_field.h \ google/protobuf/compiler/javanano/javanano_message.cc \ + google/protobuf/compiler/javanano/javanano_message.h \ + google/protobuf/compiler/javanano/javanano_message_field.cc \ google/protobuf/compiler/javanano/javanano_message_field.h \ google/protobuf/compiler/javanano/javanano_params.h \ - google/protobuf/compiler/javanano/javanano_primitive_field.h \ - google/protobuf/compiler/javanano/javanano_enum_field.cc \ - google/protobuf/compiler/javanano/javanano_enum.h \ - google/protobuf/compiler/javanano/javanano_extension.h \ - google/protobuf/compiler/javanano/javanano_field.h \ - google/protobuf/compiler/javanano/javanano_file.h \ - google/protobuf/compiler/javanano/javanano_generator.h \ - google/protobuf/compiler/javanano/javanano_helpers.h \ - google/protobuf/compiler/javanano/javanano_message_field.cc \ - google/protobuf/compiler/javanano/javanano_message.h \ google/protobuf/compiler/javanano/javanano_primitive_field.cc \ + google/protobuf/compiler/javanano/javanano_primitive_field.h \ google/protobuf/compiler/python/python_generator.cc \ google/protobuf/compiler/ruby/ruby_generator.cc @@ -305,6 +326,7 @@ EXTRA_DIST = \ google/protobuf/io/gzip_stream_unittest.sh \ google/protobuf/testdata/golden_message \ google/protobuf/testdata/golden_message_oneof_implemented \ + google/protobuf/testdata/golden_message_proto3 \ google/protobuf/testdata/golden_packed_fields_message \ google/protobuf/testdata/bad_utf8_string \ google/protobuf/testdata/text_format_unittest_data.txt \ @@ -372,7 +394,7 @@ protoc_outputs = \ google/protobuf/compiler/cpp/cpp_test_bad_identifiers.pb.cc \ google/protobuf/compiler/cpp/cpp_test_bad_identifiers.pb.h -BUILT_SOURCES = $(protoc_outputs) +BUILT_SOURCES = $(public_config) $(protoc_outputs) if USE_EXTERNAL_PROTOC @@ -458,6 +480,7 @@ protobuf_test_SOURCES = \ google/protobuf/compiler/java/java_plugin_unittest.cc \ google/protobuf/compiler/java/java_doc_comment_unittest.cc \ google/protobuf/compiler/python/python_plugin_unittest.cc \ + google/protobuf/compiler/ruby/ruby_generator_unittest.cc \ $(COMMON_TEST_SOURCES) nodist_protobuf_test_SOURCES = $(protoc_outputs) diff --git a/src/google/protobuf/any.proto b/src/google/protobuf/any.proto new file mode 100644 index 00000000..bf2aa0a9 --- /dev/null +++ b/src/google/protobuf/any.proto @@ -0,0 +1,98 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +syntax = "proto3"; + +package google.protobuf; + +option java_generate_equals_and_hash = true; +option java_multiple_files = true; +option java_outer_classname = "AnyProto"; +option java_package = "com.google.protobuf"; + + +// `Any` contains an arbitrary serialized message along with a URL +// that describes the type of the serialized message. +// +// +// +// # JSON +// The JSON representation of an `Any` value uses the regular +// representation of the deserialized, embedded message, with an +// additional field `@type` which contains the type URL. Example: +// +// package google.profile; +// message Person { +// string first_name = 1; +// string last_name = 2; +// } +// +// { +// "@type": "type.googleapis.com/google.profile.Person", +// "firstName": <string>, +// "lastName": <string> +// } +// +// If the embedded message type is well-known and has a custom JSON +// representation, that representation will be embedded adding a field +// `value` which holds the custom JSON in addition to the the `@type` +// field. Example (for message [google.protobuf.Duration][google.protobuf.Duration]): +// +// { +// "@type": "type.googleapis.com/google.protobuf.Duration", +// "value": "1.212s" +// } +// +message Any { + // A URL/resource name whose content describes the type of the + // serialized message. + // + // For URLs which use the schema `http`, `https`, or no schema, the + // following restrictions and interpretations apply: + // + // * If no schema is provided, https is assumed. + // * The last segment of the URL's path must represent the fully + // qualified name of the type (as in `path/google.protobuf.Duration`). + // * An HTTP GET on the URL must yield a [google.protobuf.Type][google.protobuf.Type] + // value in binary format, or produce an error. + // * Applications are allowed to cache lookup results based on the + // URL, or have them precompiled into a binary to avoid any + // lookup. Therefore, binary compatibility need to be preserved + // on changes to types. (Use versioned type names to manage + // breaking changes.) + // + // Schemas other than `http`, `https` (or the empty schema) might be + // used with implementation specific semantics. + // + // + string type_url = 1; + + // Must be valid serialized data of the above specified type. + bytes value = 2; +} diff --git a/src/google/protobuf/arena.h b/src/google/protobuf/arena.h index d0cb163c..4f9e39e1 100644 --- a/src/google/protobuf/arena.h +++ b/src/google/protobuf/arena.h @@ -313,6 +313,8 @@ class LIBPROTOBUF_EXPORT Arena { static const size_t kHeaderSize = sizeof(Block); static google::protobuf::internal::SequenceNumber lifecycle_id_generator_; #ifdef PROTOBUF_USE_DLLS + // Thread local variables cannot be exposed through DLL interface but we can + // wrap them in static functions. static ThreadCache& thread_cache(); #else static GOOGLE_THREAD_LOCAL ThreadCache thread_cache_; diff --git a/src/google/protobuf/arena_unittest.cc b/src/google/protobuf/arena_unittest.cc index 9d3d3e3e..390141a3 100644 --- a/src/google/protobuf/arena_unittest.cc +++ b/src/google/protobuf/arena_unittest.cc @@ -390,7 +390,7 @@ TEST(ArenaTest, ReleaseFromArenaMessageUsingReflectionMakesCopy) { const Reflection* r = arena_message->GetReflection(); const FieldDescriptor* f = arena_message->GetDescriptor()->FindFieldByName( "optional_nested_message"); - nested_msg = dynamic_cast<TestAllTypes::NestedMessage*>( + nested_msg = static_cast<TestAllTypes::NestedMessage*>( r->ReleaseMessage(arena_message, f)); } EXPECT_EQ(42, nested_msg->bb()); @@ -853,7 +853,7 @@ TEST(ArenaTest, MutableMessageReflection) { const Descriptor* d = message->GetDescriptor(); const FieldDescriptor* field = d->FindFieldByName("optional_nested_message"); TestAllTypes::NestedMessage* submessage = - dynamic_cast<TestAllTypes::NestedMessage*>( + static_cast<TestAllTypes::NestedMessage*>( r->MutableMessage(message, field)); TestAllTypes::NestedMessage* submessage_expected = message->mutable_optional_nested_message(); @@ -862,7 +862,7 @@ TEST(ArenaTest, MutableMessageReflection) { EXPECT_EQ(&arena, submessage->GetArena()); const FieldDescriptor* oneof_field = d->FindFieldByName("oneof_nested_message"); - submessage = dynamic_cast<TestAllTypes::NestedMessage*>( + submessage = static_cast<TestAllTypes::NestedMessage*>( r->MutableMessage(message, oneof_field)); submessage_expected = message->mutable_oneof_nested_message(); @@ -921,8 +921,8 @@ TEST(ArenaTest, MessageLiteOnArena) { options.initial_block = &arena_block[0]; options.initial_block_size = arena_block.size(); Arena arena(options); - const google::protobuf::MessageLite* prototype = dynamic_cast< - const google::protobuf::MessageLite*>(&TestAllTypes::default_instance()); + const google::protobuf::MessageLite* prototype = + &TestAllTypes::default_instance(); TestAllTypes initial_message; FillArenaAwareFields(&initial_message); @@ -935,8 +935,7 @@ TEST(ArenaTest, MessageLiteOnArena) { EXPECT_TRUE(generic_message != NULL); EXPECT_EQ(&arena, generic_message->GetArena()); EXPECT_TRUE(generic_message->ParseFromString(serialized)); - TestAllTypes* deserialized = dynamic_cast<TestAllTypes*>(generic_message); - EXPECT_TRUE(deserialized != NULL); + TestAllTypes* deserialized = static_cast<TestAllTypes*>(generic_message); EXPECT_EQ(42, deserialized->optional_int32()); } diff --git a/src/google/protobuf/compiler/command_line_interface_unittest.cc b/src/google/protobuf/compiler/command_line_interface_unittest.cc index dbaaa405..64e877a3 100644 --- a/src/google/protobuf/compiler/command_line_interface_unittest.cc +++ b/src/google/protobuf/compiler/command_line_interface_unittest.cc @@ -64,6 +64,10 @@ #include <gtest/gtest.h> +// Disable the whole test when we use tcmalloc for "draconian" heap checks, in +// which case tcmalloc will print warnings that fail the plugin tests. +#if !GOOGLE_PROTOBUF_HEAP_CHECK_DRACONIAN + namespace google { namespace protobuf { namespace compiler { @@ -1663,3 +1667,5 @@ TEST_F(EncodeDecodeTest, ProtoParseError) { } // namespace compiler } // namespace protobuf } // namespace google + +#endif // !GOOGLE_PROTOBUF_HEAP_CHECK_DRACONIAN diff --git a/src/google/protobuf/compiler/java/java_message_field.cc b/src/google/protobuf/compiler/java/java_message_field.cc index 538f1248..a2d12a38 100644 --- a/src/google/protobuf/compiler/java/java_message_field.cc +++ b/src/google/protobuf/compiler/java/java_message_field.cc @@ -718,6 +718,7 @@ GenerateBuilderMembers(io::Printer* printer) const { " $oneof_name$_ = null;\n" " }\n" " $set_oneof_case_message$;\n" + " $on_changed$;\n" " return $name$Builder_;\n" "}\n"); } diff --git a/src/google/protobuf/compiler/javanano/javanano_field.cc b/src/google/protobuf/compiler/javanano/javanano_field.cc index e3e4cefe..85257f3f 100644 --- a/src/google/protobuf/compiler/javanano/javanano_field.cc +++ b/src/google/protobuf/compiler/javanano/javanano_field.cc @@ -36,6 +36,7 @@ #include <google/protobuf/compiler/javanano/javanano_helpers.h> #include <google/protobuf/compiler/javanano/javanano_primitive_field.h> #include <google/protobuf/compiler/javanano/javanano_enum_field.h> +#include <google/protobuf/compiler/javanano/javanano_map_field.h> #include <google/protobuf/compiler/javanano/javanano_message_field.h> #include <google/protobuf/stubs/common.h> @@ -97,12 +98,24 @@ FieldGenerator* FieldGeneratorMap::MakeGenerator(const FieldDescriptor* field, if (field->is_repeated()) { switch (java_type) { case JAVATYPE_MESSAGE: - return new RepeatedMessageFieldGenerator(field, params); + if (IsMapEntry(field->message_type())) { + return new MapFieldGenerator(field, params); + } else { + return new RepeatedMessageFieldGenerator(field, params); + } case JAVATYPE_ENUM: return new RepeatedEnumFieldGenerator(field, params); default: return new RepeatedPrimitiveFieldGenerator(field, params); } + } else if (field->containing_oneof()) { + switch (java_type) { + case JAVATYPE_MESSAGE: + return new MessageOneofFieldGenerator(field, params); + case JAVATYPE_ENUM: + default: + return new PrimitiveOneofFieldGenerator(field, params); + } } else if (params.optional_field_accessors() && field->is_optional() && java_type != JAVATYPE_MESSAGE) { // We need a has-bit for each primitive/enum field because their default @@ -137,6 +150,59 @@ const FieldGenerator& FieldGeneratorMap::get( return *field_generators_[field->index()]; } +void SetCommonOneofVariables(const FieldDescriptor* descriptor, + map<string, string>* variables) { + (*variables)["oneof_name"] = + UnderscoresToCamelCase(descriptor->containing_oneof()); + (*variables)["oneof_capitalized_name"] = + UnderscoresToCapitalizedCamelCase(descriptor->containing_oneof()); + (*variables)["oneof_index"] = + SimpleItoa(descriptor->containing_oneof()->index()); + (*variables)["set_oneof_case"] = + "this." + (*variables)["oneof_name"] + + "Case_ = " + SimpleItoa(descriptor->number()); + (*variables)["clear_oneof_case"] = + "this." + (*variables)["oneof_name"] + "Case_ = 0"; + (*variables)["has_oneof_case"] = + "this." + (*variables)["oneof_name"] + "Case_ == " + + SimpleItoa(descriptor->number()); +} + +void GenerateOneofFieldEquals(const FieldDescriptor* descriptor, + const map<string, string>& variables, + io::Printer* printer) { + if (GetJavaType(descriptor) == JAVATYPE_BYTES) { + printer->Print(variables, + "if (this.has$capitalized_name$()) {\n" + " if (!java.util.Arrays.equals((byte[]) this.$oneof_name$_,\n" + " (byte[]) other.$oneof_name$_)) {\n" + " return false;\n" + " }\n" + "}\n"); + } else { + printer->Print(variables, + "if (this.has$capitalized_name$()) {\n" + " if (!this.$oneof_name$_.equals(other.$oneof_name$_)) {\n" + " return false;\n" + " }\n" + "}\n"); + } +} + +void GenerateOneofFieldHashCode(const FieldDescriptor* descriptor, + const map<string, string>& variables, + io::Printer* printer) { + if (GetJavaType(descriptor) == JAVATYPE_BYTES) { + printer->Print(variables, + "result = 31 * result + ($has_oneof_case$\n" + " ? java.util.Arrays.hashCode((byte[]) this.$oneof_name$_) : 0);\n"); + } else { + printer->Print(variables, + "result = 31 * result +\n" + " ($has_oneof_case$ ? this.$oneof_name$_.hashCode() : 0);\n"); + } +} + } // namespace javanano } // namespace compiler } // namespace protobuf diff --git a/src/google/protobuf/compiler/javanano/javanano_field.h b/src/google/protobuf/compiler/javanano/javanano_field.h index 6170c2c0..c2cf091c 100644 --- a/src/google/protobuf/compiler/javanano/javanano_field.h +++ b/src/google/protobuf/compiler/javanano/javanano_field.h @@ -35,6 +35,7 @@ #ifndef GOOGLE_PROTOBUF_COMPILER_JAVANANO_FIELD_H__ #define GOOGLE_PROTOBUF_COMPILER_JAVANANO_FIELD_H__ +#include <map> #include <string> #include <google/protobuf/stubs/common.h> #include <google/protobuf/descriptor.h> @@ -111,6 +112,15 @@ class FieldGeneratorMap { GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(FieldGeneratorMap); }; +void SetCommonOneofVariables(const FieldDescriptor* descriptor, + map<string, string>* variables); +void GenerateOneofFieldEquals(const FieldDescriptor* descriptor, + const map<string, string>& variables, + io::Printer* printer); +void GenerateOneofFieldHashCode(const FieldDescriptor* descriptor, + const map<string, string>& variables, + io::Printer* printer); + } // namespace javanano } // namespace compiler } // namespace protobuf diff --git a/src/google/protobuf/compiler/javanano/javanano_helpers.cc b/src/google/protobuf/compiler/javanano/javanano_helpers.cc index 2149418a..0d2ae9db 100644 --- a/src/google/protobuf/compiler/javanano/javanano_helpers.cc +++ b/src/google/protobuf/compiler/javanano/javanano_helpers.cc @@ -154,6 +154,14 @@ string UnderscoresToCamelCase(const MethodDescriptor* method) { return UnderscoresToCamelCaseImpl(method->name(), false); } +string UnderscoresToCamelCase(const OneofDescriptor* oneof) { + return UnderscoresToCamelCaseImpl(oneof->name(), false); +} + +string UnderscoresToCapitalizedCamelCase(const OneofDescriptor* oneof) { + return UnderscoresToCamelCaseImpl(oneof->name(), true); +} + string RenameJavaKeywords(const string& input) { return sRenameKeywords.RenameJavaKeywordsImpl(input); } @@ -560,6 +568,17 @@ void SetBitOperationVariables(const string name, (*variables)["different_" + name] = GenerateDifferentBit(bitIndex); } +bool HasMapField(const Descriptor* descriptor) { + for (int i = 0; i < descriptor->field_count(); ++i) { + const FieldDescriptor* field = descriptor->field(i); + if (field->type() == FieldDescriptor::TYPE_MESSAGE && + IsMapEntry(field->message_type())) { + return true; + } + } + return false; +} + } // namespace javanano } // namespace compiler } // namespace protobuf diff --git a/src/google/protobuf/compiler/javanano/javanano_helpers.h b/src/google/protobuf/compiler/javanano/javanano_helpers.h index 29310743..014c85ae 100644 --- a/src/google/protobuf/compiler/javanano/javanano_helpers.h +++ b/src/google/protobuf/compiler/javanano/javanano_helpers.h @@ -54,7 +54,9 @@ extern const char kThinSeparator[]; // Converts the field's name to camel-case, e.g. "foo_bar_baz" becomes // "fooBarBaz" or "FooBarBaz", respectively. string UnderscoresToCamelCase(const FieldDescriptor* field); +string UnderscoresToCamelCase(const OneofDescriptor* oneof); string UnderscoresToCapitalizedCamelCase(const FieldDescriptor* field); +string UnderscoresToCapitalizedCamelCase(const OneofDescriptor* oneof); // Appends an "_" to the end of a field where the name is a reserved java // keyword. For example int32 public = 1 will generate int public_. @@ -181,6 +183,14 @@ string GenerateDifferentBit(int bit_index); void SetBitOperationVariables(const string name, int bitIndex, map<string, string>* variables); +inline bool IsMapEntry(const Descriptor* descriptor) { + // TODO(liujisi): Add an option to turn on maps for proto2 syntax as well. + return descriptor->options().map_entry() && + descriptor->file()->syntax() == FileDescriptor::SYNTAX_PROTO3; +} + +bool HasMapField(const Descriptor* descriptor); + } // namespace javanano } // namespace compiler } // namespace protobuf diff --git a/src/google/protobuf/compiler/javanano/javanano_map_field.cc b/src/google/protobuf/compiler/javanano/javanano_map_field.cc new file mode 100644 index 00000000..83b2b0ce --- /dev/null +++ b/src/google/protobuf/compiler/javanano/javanano_map_field.cc @@ -0,0 +1,186 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// http://code.google.com/p/protobuf/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <google/protobuf/compiler/javanano/javanano_map_field.h> +#include <google/protobuf/compiler/javanano/javanano_helpers.h> +#include <google/protobuf/stubs/common.h> +#include <google/protobuf/io/printer.h> +#include <google/protobuf/wire_format.h> +#include <google/protobuf/stubs/strutil.h> + +namespace google { +namespace protobuf { +namespace compiler { +namespace javanano { + +namespace { + +string TypeName(const Params& params, const FieldDescriptor* field, + bool boxed) { + JavaType java_type = GetJavaType(field); + switch (java_type) { + case JAVATYPE_MESSAGE: + return ClassName(params, field->message_type()); + case JAVATYPE_INT: + case JAVATYPE_LONG: + case JAVATYPE_FLOAT: + case JAVATYPE_DOUBLE: + case JAVATYPE_BOOLEAN: + case JAVATYPE_STRING: + case JAVATYPE_BYTES: + case JAVATYPE_ENUM: + if (boxed) { + return BoxedPrimitiveTypeName(java_type); + } else { + return PrimitiveTypeName(java_type); + } + // No default because we want the compiler to complain if any new JavaTypes + // are added.. + } + + GOOGLE_LOG(FATAL) << "should not reach here."; + return ""; +} + +const FieldDescriptor* KeyField(const FieldDescriptor* descriptor) { + GOOGLE_CHECK_EQ(FieldDescriptor::TYPE_MESSAGE, descriptor->type()); + const Descriptor* message = descriptor->message_type(); + GOOGLE_CHECK(message->options().map_entry()); + return message->FindFieldByName("key"); +} + +const FieldDescriptor* ValueField(const FieldDescriptor* descriptor) { + GOOGLE_CHECK_EQ(FieldDescriptor::TYPE_MESSAGE, descriptor->type()); + const Descriptor* message = descriptor->message_type(); + GOOGLE_CHECK(message->options().map_entry()); + return message->FindFieldByName("value"); +} + +void SetMapVariables(const Params& params, + const FieldDescriptor* descriptor, map<string, string>* variables) { + const FieldDescriptor* key = KeyField(descriptor); + const FieldDescriptor* value = ValueField(descriptor); + (*variables)["name"] = + RenameJavaKeywords(UnderscoresToCamelCase(descriptor)); + (*variables)["number"] = SimpleItoa(descriptor->number()); + (*variables)["key_type"] = TypeName(params, key, false); + (*variables)["boxed_key_type"] = TypeName(params,key, true); + (*variables)["key_desc_type"] = + "TYPE_" + ToUpper(FieldDescriptor::TypeName(key->type())); + (*variables)["key_tag"] = SimpleItoa(internal::WireFormat::MakeTag(key)); + (*variables)["value_type"] = TypeName(params, value, false); + (*variables)["boxed_value_type"] = TypeName(params, value, true); + (*variables)["value_desc_type"] = + "TYPE_" + ToUpper(FieldDescriptor::TypeName(value->type())); + (*variables)["value_tag"] = SimpleItoa(internal::WireFormat::MakeTag(value)); + (*variables)["type_parameters"] = + (*variables)["boxed_key_type"] + ", " + (*variables)["boxed_value_type"]; + (*variables)["value_default"] = + value->type() == FieldDescriptor::TYPE_MESSAGE + ? "new " + (*variables)["value_type"] + "()" + : "null"; +} +} // namespace + +// =================================================================== +MapFieldGenerator::MapFieldGenerator(const FieldDescriptor* descriptor, + const Params& params) + : FieldGenerator(params), descriptor_(descriptor) { + SetMapVariables(params, descriptor, &variables_); +} + +MapFieldGenerator::~MapFieldGenerator() {} + +void MapFieldGenerator:: +GenerateMembers(io::Printer* printer, bool /* unused lazy_init */) const { + printer->Print(variables_, + "public java.util.Map<$type_parameters$> $name$;\n"); +} + +void MapFieldGenerator:: +GenerateClearCode(io::Printer* printer) const { + printer->Print(variables_, + "$name$ = null;\n"); +} + +void MapFieldGenerator:: +GenerateMergingCode(io::Printer* printer) const { + printer->Print(variables_, + "this.$name$ = com.google.protobuf.nano.InternalNano.mergeMapEntry(\n" + " input, this.$name$, mapFactory,\n" + " com.google.protobuf.nano.InternalNano.$key_desc_type$,\n" + " com.google.protobuf.nano.InternalNano.$value_desc_type$,\n" + " $value_default$,\n" + " $key_tag$, $value_tag$);\n" + "\n"); +} + +void MapFieldGenerator:: +GenerateSerializationCode(io::Printer* printer) const { + printer->Print(variables_, + "if (this.$name$ != null) {\n" + " com.google.protobuf.nano.InternalNano.serializeMapField(\n" + " output, this.$name$, $number$,\n" + " com.google.protobuf.nano.InternalNano.$key_desc_type$,\n" + " com.google.protobuf.nano.InternalNano.$value_desc_type$);\n" + "}\n"); +} + +void MapFieldGenerator:: +GenerateSerializedSizeCode(io::Printer* printer) const { + printer->Print(variables_, + "if (this.$name$ != null) {\n" + " size += com.google.protobuf.nano.InternalNano.computeMapFieldSize(\n" + " this.$name$, $number$,\n" + " com.google.protobuf.nano.InternalNano.$key_desc_type$,\n" + " com.google.protobuf.nano.InternalNano.$value_desc_type$);\n" + "}\n"); +} + +void MapFieldGenerator:: +GenerateEqualsCode(io::Printer* printer) const { + printer->Print(variables_, + "if (!com.google.protobuf.nano.InternalNano.equals(\n" + " this.$name$, other.$name$)) {\n" + " return false;\n" + "}\n"); +} + +void MapFieldGenerator:: +GenerateHashCodeCode(io::Printer* printer) const { + printer->Print(variables_, + "result = 31 * result +\n" + " com.google.protobuf.nano.InternalNano.hashCode(this.$name$);\n"); +} + +} // namespace javanano +} // namespace compiler +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/compiler/javanano/javanano_map_field.h b/src/google/protobuf/compiler/javanano/javanano_map_field.h new file mode 100644 index 00000000..c01bde38 --- /dev/null +++ b/src/google/protobuf/compiler/javanano/javanano_map_field.h @@ -0,0 +1,70 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// http://code.google.com/p/protobuf/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_COMPILER_JAVANANO_MAP_FIELD_H__ +#define GOOGLE_PROTOBUF_COMPILER_JAVANANO_MAP_FIELD_H__ + +#include <map> +#include <string> +#include <vector> +#include <google/protobuf/compiler/javanano/javanano_field.h> + +namespace google { +namespace protobuf { +namespace compiler { +namespace javanano { + +class MapFieldGenerator : public FieldGenerator { + public: + explicit MapFieldGenerator( + const FieldDescriptor* descriptor, const Params& params); + ~MapFieldGenerator(); + + // implements FieldGenerator --------------------------------------- + void GenerateMembers(io::Printer* printer, bool lazy_init) const; + void GenerateClearCode(io::Printer* printer) const; + void GenerateMergingCode(io::Printer* printer) const; + void GenerateSerializationCode(io::Printer* printer) const; + void GenerateSerializedSizeCode(io::Printer* printer) const; + void GenerateEqualsCode(io::Printer* printer) const; + void GenerateHashCodeCode(io::Printer* printer) const; + + private: + const FieldDescriptor* descriptor_; + map<string, string> variables_; + + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(MapFieldGenerator); +}; + +} // namespace javanano +} // namespace compiler +} // namespace protobuf +} // namespace google +#endif // GOOGLE_PROTOBUF_COMPILER_JAVANANO_MAP_FIELD_H__ diff --git a/src/google/protobuf/compiler/javanano/javanano_message.cc b/src/google/protobuf/compiler/javanano/javanano_message.cc index 7c52ca31..707f6b84 100644 --- a/src/google/protobuf/compiler/javanano/javanano_message.cc +++ b/src/google/protobuf/compiler/javanano/javanano_message.cc @@ -90,6 +90,7 @@ void MessageGenerator::GenerateStaticVariables(io::Printer* printer) { // Generate static members for all nested types. for (int i = 0; i < descriptor_->nested_type_count(); i++) { // TODO(kenton): Reuse MessageGenerator objects? + if (IsMapEntry(descriptor_->nested_type(i))) continue; MessageGenerator(descriptor_->nested_type(i), params_) .GenerateStaticVariables(printer); } @@ -100,6 +101,7 @@ void MessageGenerator::GenerateStaticVariableInitializers( // Generate static member initializers for all nested types. for (int i = 0; i < descriptor_->nested_type_count(); i++) { // TODO(kenton): Reuse MessageGenerator objects? + if (IsMapEntry(descriptor_->nested_type(i))) continue; MessageGenerator(descriptor_->nested_type(i), params_) .GenerateStaticVariableInitializers(printer); } @@ -159,9 +161,44 @@ void MessageGenerator::Generate(io::Printer* printer) { } for (int i = 0; i < descriptor_->nested_type_count(); i++) { + if (IsMapEntry(descriptor_->nested_type(i))) continue; MessageGenerator(descriptor_->nested_type(i), params_).Generate(printer); } + // oneof + map<string, string> vars; + vars["message_name"] = descriptor_->name(); + for (int i = 0; i < descriptor_->oneof_decl_count(); i++) { + const OneofDescriptor* oneof_desc = descriptor_->oneof_decl(i); + vars["oneof_name"] = UnderscoresToCamelCase(oneof_desc); + vars["oneof_capitalized_name"] = + UnderscoresToCapitalizedCamelCase(oneof_desc); + vars["oneof_index"] = SimpleItoa(oneof_desc->index()); + // Oneof Constants + for (int j = 0; j < oneof_desc->field_count(); j++) { + const FieldDescriptor* field = oneof_desc->field(j); + vars["number"] = SimpleItoa(field->number()); + vars["cap_field_name"] = ToUpper(field->name()); + printer->Print(vars, + "public static final int $cap_field_name$_FIELD_NUMBER = $number$;\n"); + } + // oneofCase_ and oneof_ + printer->Print(vars, + "private int $oneof_name$Case_ = 0;\n" + "private java.lang.Object $oneof_name$_;\n"); + printer->Print(vars, + "public int get$oneof_capitalized_name$Case() {\n" + " return this.$oneof_name$Case_;\n" + "}\n"); + // Oneof clear + printer->Print(vars, + "public $message_name$ clear$oneof_capitalized_name$() {\n" + " this.$oneof_name$Case_ = 0;\n" + " this.$oneof_name$_ = null;\n" + " return this;\n" + "}\n"); + } + // Lazy initialization of otherwise static final fields can help prevent the // class initializer from being generated. We want to prevent it because it // stops ProGuard from inlining any methods in this class into call sites and @@ -342,6 +379,11 @@ void MessageGenerator::GenerateMergeFromMethods(io::Printer* printer) { "classname", descriptor_->name()); printer->Indent(); + if (HasMapField(descriptor_)) { + printer->Print( + "com.google.protobuf.nano.MapFactories.MapFactory mapFactory =\n" + " com.google.protobuf.nano.MapFactories.getMapFactory();\n"); + } printer->Print( "while (true) {\n"); @@ -466,6 +508,14 @@ void MessageGenerator::GenerateClear(io::Printer* printer) { field_generators_.get(field).GenerateClearCode(printer); } + // Clear oneofs. + for (int i = 0; i < descriptor_->oneof_decl_count(); i++) { + printer->Print( + "clear$oneof_capitalized_name$();\n", + "oneof_capitalized_name", UnderscoresToCapitalizedCamelCase( + descriptor_->oneof_decl(i))); + } + // Clear unknown fields. if (params_.store_unknown_fields()) { printer->Print("unknownFieldData = null;\n"); @@ -501,6 +551,16 @@ void MessageGenerator::GenerateEquals(io::Printer* printer) { "$classname$ other = ($classname$) o;\n", "classname", descriptor_->name()); + // Checking oneof case before checking each oneof field. + for (int i = 0; i < descriptor_->oneof_decl_count(); i++) { + const OneofDescriptor* oneof_desc = descriptor_->oneof_decl(i); + printer->Print( + "if (this.$oneof_name$Case_ != other.$oneof_name$Case_) {\n" + " return false;\n" + "}\n", + "oneof_name", UnderscoresToCamelCase(oneof_desc)); + } + for (int i = 0; i < descriptor_->field_count(); i++) { const FieldDescriptor* field = descriptor_->field(i); field_generators_.get(field).GenerateEqualsCode(printer); diff --git a/src/google/protobuf/compiler/javanano/javanano_message_field.cc b/src/google/protobuf/compiler/javanano/javanano_message_field.cc index a46081d0..181c4060 100644 --- a/src/google/protobuf/compiler/javanano/javanano_message_field.cc +++ b/src/google/protobuf/compiler/javanano/javanano_message_field.cc @@ -146,12 +146,87 @@ GenerateHashCodeCode(io::Printer* printer) const { "result = 31 * result +\n" " (this.$name$ == null ? 0 : this.$name$.hashCode());\n"); } +// =================================================================== + +MessageOneofFieldGenerator::MessageOneofFieldGenerator( + const FieldDescriptor* descriptor, const Params& params) + : FieldGenerator(params), descriptor_(descriptor) { + SetMessageVariables(params, descriptor, &variables_); + SetCommonOneofVariables(descriptor, &variables_); +} + +MessageOneofFieldGenerator::~MessageOneofFieldGenerator() {} + +void MessageOneofFieldGenerator:: +GenerateMembers(io::Printer* printer, bool /* unused lazy_init */) const { + printer->Print(variables_, + "public boolean has$capitalized_name$() {\n" + " return $has_oneof_case$;\n" + "}\n" + "public $type$ get$capitalized_name$() {\n" + " if ($has_oneof_case$) {\n" + " return ($type$) this.$oneof_name$_;\n" + " }\n" + " return null;\n" + "}\n" + "public $message_name$ set$capitalized_name$($type$ value) {\n" + " if (value == null) { throw new java.lang.NullPointerException(); }\n" + " $set_oneof_case$;\n" + " this.$oneof_name$_ = value;\n" + " return this;\n" + "}\n"); +} + +void MessageOneofFieldGenerator:: +GenerateClearCode(io::Printer* printer) const { + // No clear method for oneof fields. +} + +void MessageOneofFieldGenerator:: +GenerateMergingCode(io::Printer* printer) const { + printer->Print(variables_, + "if (!($has_oneof_case$)) {\n" + " this.$oneof_name$_ = new $type$();\n" + "}\n" + "input.readMessage(\n" + " (com.google.protobuf.nano.MessageNano) this.$oneof_name$_);\n" + "$set_oneof_case$;\n"); +} + +void MessageOneofFieldGenerator:: +GenerateSerializationCode(io::Printer* printer) const { + printer->Print(variables_, + "if ($has_oneof_case$) {\n" + " output.writeMessage($number$,\n" + " (com.google.protobuf.nano.MessageNano) this.$oneof_name$_);\n" + "}\n"); +} + +void MessageOneofFieldGenerator:: +GenerateSerializedSizeCode(io::Printer* printer) const { + printer->Print(variables_, + "if ($has_oneof_case$) {\n" + " size += com.google.protobuf.nano.CodedOutputByteBufferNano\n" + " .computeMessageSize($number$,\n" + " (com.google.protobuf.nano.MessageNano) this.$oneof_name$_);\n" + "}\n"); +} + +void MessageOneofFieldGenerator:: +GenerateEqualsCode(io::Printer* printer) const { + GenerateOneofFieldEquals(descriptor_, variables_, printer); +} + +void MessageOneofFieldGenerator:: +GenerateHashCodeCode(io::Printer* printer) const { + GenerateOneofFieldHashCode(descriptor_, variables_, printer); +} // =================================================================== -RepeatedMessageFieldGenerator:: -RepeatedMessageFieldGenerator(const FieldDescriptor* descriptor, const Params& params) - : FieldGenerator(params), descriptor_(descriptor) { +RepeatedMessageFieldGenerator::RepeatedMessageFieldGenerator( + const FieldDescriptor* descriptor, const Params& params) + : FieldGenerator(params), descriptor_(descriptor) { SetMessageVariables(params, descriptor, &variables_); } diff --git a/src/google/protobuf/compiler/javanano/javanano_message_field.h b/src/google/protobuf/compiler/javanano/javanano_message_field.h index 5d35fd24..6c615f5e 100644 --- a/src/google/protobuf/compiler/javanano/javanano_message_field.h +++ b/src/google/protobuf/compiler/javanano/javanano_message_field.h @@ -66,6 +66,28 @@ class MessageFieldGenerator : public FieldGenerator { GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(MessageFieldGenerator); }; +class MessageOneofFieldGenerator : public FieldGenerator { + public: + explicit MessageOneofFieldGenerator(const FieldDescriptor* descriptor, + const Params& params); + ~MessageOneofFieldGenerator(); + + // implements FieldGenerator --------------------------------------- + void GenerateMembers(io::Printer* printer, bool lazy_init) const; + void GenerateClearCode(io::Printer* printer) const; + void GenerateMergingCode(io::Printer* printer) const; + void GenerateSerializationCode(io::Printer* printer) const; + void GenerateSerializedSizeCode(io::Printer* printer) const; + void GenerateEqualsCode(io::Printer* printer) const; + void GenerateHashCodeCode(io::Printer* printer) const; + + private: + const FieldDescriptor* descriptor_; + map<string, string> variables_; + + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(MessageOneofFieldGenerator); +}; + class RepeatedMessageFieldGenerator : public FieldGenerator { public: explicit RepeatedMessageFieldGenerator(const FieldDescriptor* descriptor, diff --git a/src/google/protobuf/compiler/javanano/javanano_primitive_field.cc b/src/google/protobuf/compiler/javanano/javanano_primitive_field.cc index 919a2f89..41bad0a6 100644 --- a/src/google/protobuf/compiler/javanano/javanano_primitive_field.cc +++ b/src/google/protobuf/compiler/javanano/javanano_primitive_field.cc @@ -685,8 +685,79 @@ GenerateHashCodeCode(io::Printer* printer) const { // =================================================================== -RepeatedPrimitiveFieldGenerator:: -RepeatedPrimitiveFieldGenerator(const FieldDescriptor* descriptor, const Params& params) +PrimitiveOneofFieldGenerator::PrimitiveOneofFieldGenerator( + const FieldDescriptor* descriptor, const Params& params) + : FieldGenerator(params), descriptor_(descriptor) { + SetPrimitiveVariables(descriptor, params, &variables_); + SetCommonOneofVariables(descriptor, &variables_); +} + +PrimitiveOneofFieldGenerator::~PrimitiveOneofFieldGenerator() {} + +void PrimitiveOneofFieldGenerator::GenerateMembers( + io::Printer* printer, bool /*unused lazy_init*/) const { + printer->Print(variables_, + "public boolean has$capitalized_name$() {\n" + " return $has_oneof_case$;\n" + "}\n" + "public $type$ get$capitalized_name$() {\n" + " if ($has_oneof_case$) {\n" + " return ($type$) ($boxed_type$) this.$oneof_name$_;\n" + " }\n" + " return $default$;\n" + "}\n" + "public $message_name$ set$capitalized_name$($type$ value) {\n" + " $set_oneof_case$;\n" + " this.$oneof_name$_ = value;\n" + " return this;\n" + "}\n"); +} + +void PrimitiveOneofFieldGenerator::GenerateClearCode( + io::Printer* printer) const { + // No clear method for oneof fields. +} + +void PrimitiveOneofFieldGenerator::GenerateMergingCode( + io::Printer* printer) const { + printer->Print(variables_, + "this.$oneof_name$_ = input.read$capitalized_type$();\n" + "$set_oneof_case$;\n"); +} + +void PrimitiveOneofFieldGenerator::GenerateSerializationCode( + io::Printer* printer) const { + printer->Print(variables_, + "if ($has_oneof_case$) {\n" + " output.write$capitalized_type$(\n" + " $number$, ($boxed_type$) this.$oneof_name$_);\n" + "}\n"); +} + +void PrimitiveOneofFieldGenerator::GenerateSerializedSizeCode( + io::Printer* printer) const { + printer->Print(variables_, + "if ($has_oneof_case$) {\n" + " size += com.google.protobuf.nano.CodedOutputByteBufferNano\n" + " .compute$capitalized_type$Size(\n" + " $number$, ($boxed_type$) this.$oneof_name$_);\n" + "}\n"); +} + +void PrimitiveOneofFieldGenerator::GenerateEqualsCode( + io::Printer* printer) const { + GenerateOneofFieldEquals(descriptor_, variables_, printer); +} + +void PrimitiveOneofFieldGenerator::GenerateHashCodeCode( + io::Printer* printer) const { + GenerateOneofFieldHashCode(descriptor_, variables_, printer); +} + +// =================================================================== + +RepeatedPrimitiveFieldGenerator::RepeatedPrimitiveFieldGenerator( + const FieldDescriptor* descriptor, const Params& params) : FieldGenerator(params), descriptor_(descriptor) { SetPrimitiveVariables(descriptor, params, &variables_); } diff --git a/src/google/protobuf/compiler/javanano/javanano_primitive_field.h b/src/google/protobuf/compiler/javanano/javanano_primitive_field.h index c04a19b7..ca7116ff 100644 --- a/src/google/protobuf/compiler/javanano/javanano_primitive_field.h +++ b/src/google/protobuf/compiler/javanano/javanano_primitive_field.h @@ -47,7 +47,7 @@ namespace javanano { class PrimitiveFieldGenerator : public FieldGenerator { public: explicit PrimitiveFieldGenerator( - const FieldDescriptor* descriptor, const Params ¶ms); + const FieldDescriptor* descriptor, const Params& params); ~PrimitiveFieldGenerator(); // implements FieldGenerator --------------------------------------- @@ -94,9 +94,32 @@ class AccessorPrimitiveFieldGenerator : public FieldGenerator { GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(AccessorPrimitiveFieldGenerator); }; +class PrimitiveOneofFieldGenerator : public FieldGenerator { + public: + explicit PrimitiveOneofFieldGenerator( + const FieldDescriptor* descriptor, const Params& params); + ~PrimitiveOneofFieldGenerator(); + + // implements FieldGenerator --------------------------------------- + void GenerateMembers(io::Printer* printer, bool lazy_init) const; + void GenerateClearCode(io::Printer* printer) const; + void GenerateMergingCode(io::Printer* printer) const; + void GenerateSerializationCode(io::Printer* printer) const; + void GenerateSerializedSizeCode(io::Printer* printer) const; + void GenerateEqualsCode(io::Printer* printer) const; + void GenerateHashCodeCode(io::Printer* printer) const; + + private: + const FieldDescriptor* descriptor_; + map<string, string> variables_; + + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(PrimitiveOneofFieldGenerator); +}; + class RepeatedPrimitiveFieldGenerator : public FieldGenerator { public: - explicit RepeatedPrimitiveFieldGenerator(const FieldDescriptor* descriptor, const Params& params); + explicit RepeatedPrimitiveFieldGenerator(const FieldDescriptor* descriptor, + const Params& params); ~RepeatedPrimitiveFieldGenerator(); // implements FieldGenerator --------------------------------------- diff --git a/src/google/protobuf/compiler/parser_unittest.cc b/src/google/protobuf/compiler/parser_unittest.cc index 638a83b9..fe0719f8 100644 --- a/src/google/protobuf/compiler/parser_unittest.cc +++ b/src/google/protobuf/compiler/parser_unittest.cc @@ -401,13 +401,27 @@ TEST_F(ParseMessageTest, FieldDefaults) { " field { type:TYPE_BOOL default_value:\"true\" " ETC " }" " field { type_name:\"Foo\" default_value:\"FOO\" " ETC " }" - " field { type:TYPE_INT32 default_value:\"2147483647\" " ETC " }" - " field { type:TYPE_INT32 default_value:\"-2147483648\" " ETC " }" - " field { type:TYPE_UINT32 default_value:\"4294967295\" " ETC " }" - " field { type:TYPE_INT64 default_value:\"9223372036854775807\" " ETC " }" - " field { type:TYPE_INT64 default_value:\"-9223372036854775808\" " ETC " }" - " field { type:TYPE_UINT64 default_value:\"18446744073709551615\" " ETC " }" - " field { type:TYPE_DOUBLE default_value:\"43981\" " ETC " }" + " field {" + " type:TYPE_INT32 default_value:\"2147483647\" " ETC + " }" + " field {" + " type:TYPE_INT32 default_value:\"-2147483648\" " ETC + " }" + " field {" + " type:TYPE_UINT32 default_value:\"4294967295\" " ETC + " }" + " field {" + " type:TYPE_INT64 default_value:\"9223372036854775807\" " ETC + " }" + " field {" + " type:TYPE_INT64 default_value:\"-9223372036854775808\" " ETC + " }" + " field {" + " type:TYPE_UINT64 default_value:\"18446744073709551615\" " ETC + " }" + " field {" + " type:TYPE_DOUBLE default_value:\"43981\" " ETC + " }" "}"); #undef ETC } diff --git a/src/google/protobuf/compiler/python/python_generator.cc b/src/google/protobuf/compiler/python/python_generator.cc index b30d1972..7b3b5fa3 100644 --- a/src/google/protobuf/compiler/python/python_generator.cc +++ b/src/google/protobuf/compiler/python/python_generator.cc @@ -273,6 +273,19 @@ string StringifyDefaultValue(const FieldDescriptor& field) { return ""; } +string StringifySyntax(FileDescriptor::Syntax syntax) { + switch (syntax) { + case FileDescriptor::SYNTAX_PROTO2: + return "proto2"; + case FileDescriptor::SYNTAX_PROTO3: + return "proto3"; + case FileDescriptor::SYNTAX_UNKNOWN: + default: + GOOGLE_LOG(FATAL) << "Unsupported syntax; this generator only supports proto2 " + "and proto3 syntax."; + return ""; + } +} } // namespace @@ -367,10 +380,12 @@ void Generator::PrintFileDescriptor() const { m["descriptor_name"] = kDescriptorKey; m["name"] = file_->name(); m["package"] = file_->package(); + m["syntax"] = StringifySyntax(file_->syntax()); const char file_descriptor_template[] = "$descriptor_name$ = _descriptor.FileDescriptor(\n" " name='$name$',\n" - " package='$package$',\n"; + " package='$package$',\n" + " syntax='$syntax$',\n"; printer_->Print(m, file_descriptor_template); printer_->Indent(); printer_->Print( @@ -414,7 +429,7 @@ void Generator::PrintTopLevelEnums() const { for (int j = 0; j < enum_descriptor.value_count(); ++j) { const EnumValueDescriptor& value_descriptor = *enum_descriptor.value(j); top_level_enum_values.push_back( - make_pair(value_descriptor.name(), value_descriptor.number())); + std::make_pair(value_descriptor.name(), value_descriptor.number())); } } @@ -581,14 +596,15 @@ void Generator::PrintServiceDescriptor( } -void Generator::PrintDescriptorKeyAndModuleName(const ServiceDescriptor& descriptor) const { +void Generator::PrintDescriptorKeyAndModuleName( + const ServiceDescriptor& descriptor) const { printer_->Print( "$descriptor_key$ = $descriptor_name$,\n", "descriptor_key", kDescriptorKey, "descriptor_name", ModuleLevelServiceDescriptorName(descriptor)); printer_->Print( "__module__ = '$module_name$'\n", - "module_name", ModuleName(file_->name())); + "module_name", ModuleName(file_->name())); } void Generator::PrintServiceClass(const ServiceDescriptor& descriptor) const { @@ -664,10 +680,12 @@ void Generator::PrintDescriptor(const Descriptor& message_descriptor) const { message_descriptor.options().SerializeToString(&options_string); printer_->Print( "options=$options_value$,\n" - "is_extendable=$extendable$", + "is_extendable=$extendable$,\n" + "syntax='$syntax$'", "options_value", OptionsValue("MessageOptions", options_string), "extendable", message_descriptor.extension_range_count() > 0 ? - "True" : "False"); + "True" : "False", + "syntax", StringifySyntax(message_descriptor.file()->syntax())); printer_->Print(",\n"); // Extension ranges diff --git a/src/google/protobuf/compiler/python/python_generator.h b/src/google/protobuf/compiler/python/python_generator.h index ee68ad72..2ddac601 100644 --- a/src/google/protobuf/compiler/python/python_generator.h +++ b/src/google/protobuf/compiler/python/python_generator.h @@ -127,7 +127,8 @@ class LIBPROTOC_EXPORT Generator : public CodeGenerator { void PrintServiceDescriptor(const ServiceDescriptor& descriptor) const; void PrintServiceClass(const ServiceDescriptor& descriptor) const; void PrintServiceStub(const ServiceDescriptor& descriptor) const; - void PrintDescriptorKeyAndModuleName(const ServiceDescriptor& descriptor) const ; + void PrintDescriptorKeyAndModuleName( + const ServiceDescriptor& descriptor) const ; void PrintEnumValueDescriptor(const EnumValueDescriptor& descriptor) const; string OptionsValue(const string& class_name, diff --git a/src/google/protobuf/compiler/ruby/ruby_generator.cc b/src/google/protobuf/compiler/ruby/ruby_generator.cc index c5687903..a9b6837e 100644 --- a/src/google/protobuf/compiler/ruby/ruby_generator.cc +++ b/src/google/protobuf/compiler/ruby/ruby_generator.cc @@ -47,7 +47,7 @@ namespace compiler { namespace ruby { // Forward decls. -std::string IntToString(uint32_t value); +std::string IntToString(uint32 value); std::string StripDotProto(const std::string& proto_file); std::string LabelForField(google::protobuf::FieldDescriptor* field); std::string TypeName(google::protobuf::FieldDescriptor* field); @@ -64,7 +64,7 @@ void GenerateEnumAssignment( const google::protobuf::EnumDescriptor* en, google::protobuf::io::Printer* printer); -std::string IntToString(uint32_t value) { +std::string IntToString(uint32 value) { std::ostringstream os; os << value; return os.str(); @@ -100,15 +100,35 @@ std::string TypeName(const google::protobuf::FieldDescriptor* field) { } } -void GenerateMessage(const google::protobuf::Descriptor* message, - google::protobuf::io::Printer* printer) { - printer->Print( - "add_message \"$name$\" do\n", - "name", message->full_name()); - printer->Indent(); +void GenerateField(const google::protobuf::FieldDescriptor* field, + google::protobuf::io::Printer* printer) { + + if (field->is_map()) { + const FieldDescriptor* key_field = + field->message_type()->FindFieldByNumber(1); + const FieldDescriptor* value_field = + field->message_type()->FindFieldByNumber(2); + + printer->Print( + "map :$name$, :$key_type$, :$value_type$, $number$", + "name", field->name(), + "key_type", TypeName(key_field), + "value_type", TypeName(value_field), + "number", IntToString(field->number())); + + if (value_field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + printer->Print( + ", \"$subtype$\"\n", + "subtype", value_field->message_type()->full_name()); + } else if (value_field->cpp_type() == FieldDescriptor::CPPTYPE_ENUM) { + printer->Print( + ", \"$subtype$\"\n", + "subtype", value_field->enum_type()->full_name()); + } else { + printer->Print("\n"); + } + } else { - for (int i = 0; i < message->field_count(); i++) { - const FieldDescriptor* field = message->field(i); printer->Print( "$label$ :$name$, ", "label", LabelForField(field), @@ -117,6 +137,7 @@ void GenerateMessage(const google::protobuf::Descriptor* message, ":$type$, $number$", "type", TypeName(field), "number", IntToString(field->number())); + if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { printer->Print( ", \"$subtype$\"\n", @@ -129,6 +150,49 @@ void GenerateMessage(const google::protobuf::Descriptor* message, printer->Print("\n"); } } +} + +void GenerateOneof(const google::protobuf::OneofDescriptor* oneof, + google::protobuf::io::Printer* printer) { + printer->Print( + "oneof :$name$ do\n", + "name", oneof->name()); + printer->Indent(); + + for (int i = 0; i < oneof->field_count(); i++) { + const FieldDescriptor* field = oneof->field(i); + GenerateField(field, printer); + } + + printer->Outdent(); + printer->Print("end\n"); +} + +void GenerateMessage(const google::protobuf::Descriptor* message, + google::protobuf::io::Printer* printer) { + + // Don't generate MapEntry messages -- we use the Ruby extension's native + // support for map fields instead. + if (message->options().map_entry()) { + return; + } + + printer->Print( + "add_message \"$name$\" do\n", + "name", message->full_name()); + printer->Indent(); + + for (int i = 0; i < message->field_count(); i++) { + const FieldDescriptor* field = message->field(i); + if (!field->containing_oneof()) { + GenerateField(field, printer); + } + } + + for (int i = 0; i < message->oneof_decl_count(); i++) { + const OneofDescriptor* oneof = message->oneof_decl(i); + GenerateOneof(oneof, printer); + } printer->Outdent(); printer->Print("end\n"); @@ -185,6 +249,13 @@ void GenerateMessageAssignment( const std::string& prefix, const google::protobuf::Descriptor* message, google::protobuf::io::Printer* printer) { + + // Don't generate MapEntry messages -- we use the Ruby extension's native + // support for map fields instead. + if (message->options().map_entry()) { + return; + } + printer->Print( "$prefix$$name$ = ", "prefix", prefix, @@ -307,7 +378,8 @@ bool Generator::Generate( std::string filename = StripDotProto(file->name()) + ".rb"; - scoped_ptr<io::ZeroCopyOutputStream> output(generator_context->Open(filename)); + scoped_ptr<io::ZeroCopyOutputStream> output( + generator_context->Open(filename)); io::Printer printer(output.get(), '$'); GenerateFile(file, &printer); diff --git a/src/google/protobuf/compiler/ruby/ruby_generator.h b/src/google/protobuf/compiler/ruby/ruby_generator.h index 48dbefd1..75555c31 100644 --- a/src/google/protobuf/compiler/ruby/ruby_generator.h +++ b/src/google/protobuf/compiler/ruby/ruby_generator.h @@ -40,7 +40,8 @@ namespace protobuf { namespace compiler { namespace ruby { -class Generator : public google::protobuf::compiler::CodeGenerator { +class LIBPROTOC_EXPORT Generator + : public google::protobuf::compiler::CodeGenerator { virtual bool Generate( const FileDescriptor* file, const string& parameter, diff --git a/src/google/protobuf/compiler/ruby/ruby_generator_unittest.cc b/src/google/protobuf/compiler/ruby/ruby_generator_unittest.cc new file mode 100644 index 00000000..e35ca695 --- /dev/null +++ b/src/google/protobuf/compiler/ruby/ruby_generator_unittest.cc @@ -0,0 +1,119 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2014 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <memory> + +#include <google/protobuf/compiler/ruby/ruby_generator.h> +#include <google/protobuf/compiler/command_line_interface.h> +#include <google/protobuf/io/zero_copy_stream.h> +#include <google/protobuf/io/printer.h> + +#include <google/protobuf/testing/googletest.h> +#include <gtest/gtest.h> +#include <google/protobuf/testing/file.h> + +namespace google { +namespace protobuf { +namespace compiler { +namespace ruby { +namespace { + +string FindRubyTestDir() { + // Inspired by TestSourceDir() in src/google/protobuf/testing/googletest.cc. + string prefix = "."; + while (!File::Exists(prefix + "/ruby/tests")) { + if (!File::Exists(prefix)) { + GOOGLE_LOG(FATAL) + << "Could not find Ruby test directory. Please run tests from " + "somewhere within the protobuf source package."; + } + prefix += "/.."; + } + return prefix + "/ruby/tests"; +} + +// This test is a simple golden-file test over the output of the Ruby code +// generator. When we make changes to the Ruby extension and alter the Ruby code +// generator to use those changes, we should (i) manually test the output of the +// code generator with the extension, and (ii) update the golden output above. +// Some day, we may integrate build systems between protoc and the language +// extensions to the point where we can do this test in a more automated way. + +TEST(RubyGeneratorTest, GeneratorTest) { + string ruby_tests = FindRubyTestDir(); + + google::protobuf::compiler::CommandLineInterface cli; + cli.SetInputsAreProtoPathRelative(true); + + ruby::Generator ruby_generator; + cli.RegisterGenerator("--ruby_out", &ruby_generator, ""); + + // Copy generated_code.proto to the temporary test directory. + string test_input; + GOOGLE_CHECK_OK(File::GetContents( + ruby_tests + "/generated_code.proto", + &test_input, + true)); + GOOGLE_CHECK_OK(File::SetContents( + TestTempDir() + "/generated_code.proto", + test_input, + true)); + + // Invoke the proto compiler (we will be inside TestTempDir() at this point). + string ruby_out = "--ruby_out=" + TestTempDir(); + string proto_path = "--proto_path=" + TestTempDir(); + const char* argv[] = { + "protoc", + ruby_out.c_str(), + proto_path.c_str(), + "generated_code.proto", + }; + + EXPECT_EQ(0, cli.Run(4, argv)); + + // Load the generated output and compare to the expected result. + string output; + GOOGLE_CHECK_OK(File::GetContents( + TestTempDir() + "/generated_code.rb", + &output, + true)); + string expected_output; + GOOGLE_CHECK_OK(File::GetContents( + ruby_tests + "/generated_code.rb", + &expected_output, + true)); + EXPECT_EQ(expected_output, output); +} + +} // namespace +} // namespace ruby +} // namespace compiler +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/duration.proto b/src/google/protobuf/duration.proto new file mode 100644 index 00000000..868c732f --- /dev/null +++ b/src/google/protobuf/duration.proto @@ -0,0 +1,93 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +syntax = "proto3"; + +package google.protobuf; + +option java_generate_equals_and_hash = true; +option java_multiple_files = true; +option java_outer_classname = "DurationProto"; +option java_package = "com.google.protobuf"; + + +// A Duration represents a signed, fixed-length span of time represented +// as a count of seconds and fractions of seconds at nanosecond +// resolution. It is independent of any calendar and concepts like "day" +// or "month". It is related to Timestamp in that the difference between +// two Timestamp values is a Duration and it can be added or subtracted +// from a Timestamp. Range is approximately +-10,000 years. +// +// Example 1: compute Duration from two Timestamps in pseudo code. +// +// Timestamp start = ...; +// Timestamp end = ...; +// Duration duration = ...; +// +// duration.seconds = end.seconds - start.seconds; +// duration.nanos = end.nanos - start.nanos; +// +// if (duration.seconds < 0 && duration.nanos > 0) { +// duration.seconds += 1; +// duration.nanos -= 1000000000; +// } else if (durations.seconds > 0 && duration.nanos < 0) { +// duration.seconds -= 1; +// duration.nanos += 1000000000; +// } +// +// Example 2: compute Timestamp from Timestamp + Duration in pseudo code. +// +// Timestamp start = ...; +// Duration duration = ...; +// Timestamp end = ...; +// +// end.seconds = start.seconds + duration.seconds; +// end.nanos = start.nanos + duration.nanos; +// +// if (end.nanos < 0) { +// end.seconds -= 1; +// end.nanos += 1000000000; +// } else if (end.nanos >= 1000000000) { +// end.seconds += 1; +// end.nanos -= 1000000000; +// } +// +message Duration { + // Signed seconds of the span of time. Must be from -315,576,000,000 + // to +315,576,000,000 inclusive. + int64 seconds = 1; + + // Signed fractions of a second at nanosecond resolution of the span + // of time. Durations less than one second are represented with a 0 + // seconds field and a positive or negative nanos field. For durations + // of one second or more, a non-zero value for the nanos field must be + // of the same sign as the seconds field. Must be from -999,999,999 + // to +999,999,999 inclusive. + int32 nanos = 2; +} diff --git a/src/google/protobuf/field_mask.proto b/src/google/protobuf/field_mask.proto new file mode 100644 index 00000000..492d4b06 --- /dev/null +++ b/src/google/protobuf/field_mask.proto @@ -0,0 +1,161 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +syntax = "proto3"; + +package google.protobuf; + +option java_multiple_files = true; +option java_outer_classname = "FieldMaskProto"; +option java_package = "com.google.protobuf"; + + +// `FieldMask` represents a set of symbolic field paths, for example: +// +// paths: "f.a" +// paths: "f.b.d" +// +// Here `f` represents a field in some root message, `a` and `b` +// fields in the message found in `f`, and `d` a field found in the +// message in `f.b`. +// +// Field masks are used to specify a subset of fields that should be +// returned by a get operation or modified by an update operation. +// Field masks also have a custom JSON encoding (see below). +// +// # Field Masks in Projections +// When used in the context of a projection, a response message or +// sub-message is filtered by the API to only contain those fields as +// specified in the mask. For example, if the mask in the previous +// example is applied to a response message as follows: +// +// f { +// a : 22 +// b { +// d : 1 +// x : 2 +// } +// y : 13 +// } +// z: 8 +// +// The result will not contain specific values for fields x,y and z +// (there value will be set to the default, and omitted in proto text +// output): +// +// +// f { +// a : 22 +// b { +// d : 1 +// } +// } +// +// A repeated field is not allowed except at the last position of a +// field mask. +// +// If a FieldMask object is not present in a get operation, the +// operation applies to all fields (as if a FieldMask of all fields +// had been specified). +// +// Note that a field mask does not necessarily applies to the +// top-level response message. In case of a REST get operation, the +// field mask applies directly to the response, but in case of a REST +// list operation, the mask instead applies to each individual message +// in the returned resource list. In case of a REST custom method, +// other definitions may be used. Where the mask applies will be +// clearly documented together with its declaration in the API. In +// any case, the effect on the returned resource/resources is required +// behavior for APIs. +// +// # Field Masks in Update Operations +// A field mask in update operations specifies which fields of the +// targeted resource are going to be updated. The API is required +// to only change the values of the fields as specified in the mask +// and leave the others untouched. If a resource is passed in to +// describe the updated values, the API ignores the values of all +// fields not covered by the mask. +// +// In order to reset a field's value to the default, the field must +// be in the mask and set to the default value in the provided resource. +// Hence, in order to reset all fields of a resource, provide a default +// instance of the resource and set all fields in the mask, or do +// not provide a mask as described below. +// +// If a field mask is not present on update, the operation applies to +// all fields (as if a field mask of all fields has been specified). +// Note that in the presence of schema evolution, this may mean that +// fields the client does not know and has therefore not filled into +// the request will be reset to their default. If this is unwanted +// behavior, a specific service may require a client to always specify +// a field mask, producing an error if not. +// +// As with get operations, the location of the resource which +// describes the updated values in the request message depends on the +// operation kind. In any case, the effect of the field mask is +// required to be honored by the API. +// +// ## Considerations for HTTP REST +// The HTTP kind of an update operation which uses a field mask must +// be set to PATCH instead of PUT in order to satisfy HTTP semantics +// (PUT must only be used for full updates). +// +// # JSON Encoding of Field Masks +// In JSON, a field mask is encoded as a single string where paths are +// separated by a comma. Fields name in each path are converted +// to/from lower-camel naming conventions. +// +// As an example, consider the following message declarations: +// +// message Profile { +// User user = 1; +// Photo photo = 2; +// } +// message User { +// string display_name = 1; +// string address = 2; +// } +// +// In proto a field mask for `Profile` may look as such: +// +// mask { +// paths: "user.display_name" +// paths: "photo" +// } +// +// In JSON, the same mask is represented as below: +// +// { +// mask: "user.displayName,photo" +// } +// +message FieldMask { + // The set of field mask paths. + repeated string paths = 1; +} diff --git a/src/google/protobuf/generated_message_reflection.cc b/src/google/protobuf/generated_message_reflection.cc index b500b9c5..34826801 100644 --- a/src/google/protobuf/generated_message_reflection.cc +++ b/src/google/protobuf/generated_message_reflection.cc @@ -247,8 +247,14 @@ namespace { UnknownFieldSet* empty_unknown_field_set_ = NULL; GOOGLE_PROTOBUF_DECLARE_ONCE(empty_unknown_field_set_once_); +void DeleteEmptyUnknownFieldSet() { + delete empty_unknown_field_set_; + empty_unknown_field_set_ = NULL; +} + void InitEmptyUnknownFieldSet() { empty_unknown_field_set_ = new UnknownFieldSet; + internal::OnShutdown(&DeleteEmptyUnknownFieldSet); } const UnknownFieldSet& GetEmptyUnknownFieldSet() { diff --git a/src/google/protobuf/map_entry.h b/src/google/protobuf/map_entry.h index 217b15f6..30302a09 100644 --- a/src/google/protobuf/map_entry.h +++ b/src/google/protobuf/map_entry.h @@ -45,7 +45,8 @@ namespace internal { // Register all MapEntry default instances so we can delete them in // ShutdownProtobufLibrary(). -void LIBPROTOBUF_EXPORT RegisterMapEntryDefaultInstance(MessageLite* default_instance); +void LIBPROTOBUF_EXPORT RegisterMapEntryDefaultInstance( + MessageLite* default_instance); // This is the common base class for MapEntry. It is used by MapFieldBase in // reflection api, in which the static type of key and value is unknown. diff --git a/src/google/protobuf/message.cc b/src/google/protobuf/message.cc index afe95461..28955b35 100644 --- a/src/google/protobuf/message.cc +++ b/src/google/protobuf/message.cc @@ -441,6 +441,30 @@ const internal::RepeatedFieldAccessor* Reflection::RepeatedFieldAccessor( } namespace internal { +namespace { +void ShutdownRepeatedFieldAccessor() { + Singleton<internal::RepeatedFieldPrimitiveAccessor<int32> >::ShutDown(); + Singleton<internal::RepeatedFieldPrimitiveAccessor<uint32> >::ShutDown(); + Singleton<internal::RepeatedFieldPrimitiveAccessor<int64> >::ShutDown(); + Singleton<internal::RepeatedFieldPrimitiveAccessor<uint64> >::ShutDown(); + Singleton<internal::RepeatedFieldPrimitiveAccessor<float> >::ShutDown(); + Singleton<internal::RepeatedFieldPrimitiveAccessor<double> >::ShutDown(); + Singleton<internal::RepeatedFieldPrimitiveAccessor<bool> >::ShutDown(); + Singleton<internal::RepeatedPtrFieldStringAccessor>::ShutDown(); + Singleton<internal::RepeatedPtrFieldMessageAccessor>::ShutDown(); + Singleton<internal::MapFieldAccessor>::ShutDown(); +}; + +struct ShutdownRepeatedFieldRegister { + ShutdownRepeatedFieldRegister() { + OnShutdown(&ShutdownRepeatedFieldAccessor); + } +} shutdown_; + +} // namesapce +} // namespace internal + +namespace internal { // Macro defined in repeated_field.h. We can only define the Message-specific // GenericTypeHandler specializations here because we depend on Message, which // is not part of proto2-lite hence is not available in repeated_field.h. diff --git a/src/google/protobuf/struct.proto b/src/google/protobuf/struct.proto new file mode 100644 index 00000000..9f27eb43 --- /dev/null +++ b/src/google/protobuf/struct.proto @@ -0,0 +1,83 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +syntax = "proto3"; + +package google.protobuf; + +option java_generate_equals_and_hash = true; +option java_multiple_files = true; +option java_outer_classname = "StructProto"; +option java_package = "com.google.protobuf"; + + +// `Struct` represents a structured data value, consisting of fields +// which map to dynamically typed values. In some languages, `Struct` +// might be supported by a native representation. For example, in +// scripting languages like JS a struct is represented as an +// object. The details of that representation are described together +// with the proto support for the language. +message Struct { + // Map of dynamically typed values. + map<string, Value> fields = 1; +} + +// `Value` represents a dynamically typed value which can be either +// null, a number, a string, a boolean, a recursive struct value, or a +// list of values. A producer of value is expected to set one of that +// variants, absence of any variant indicates an error. +message Value { + oneof kind { + // Represents a null value. + NullValue null_value = 1; + // Represents a double value. + double number_value = 2; + // Represents a string value. + string string_value = 3; + // Represents a boolean value. + bool bool_value = 4; + // Represents a structured value. + Struct struct_value = 5; + // Represents a repeated `Value`. + ListValue list_value = 6; + } +} + +// `NullValue` is a singleton enumeration to represent the null +// value for the `Value` type union. +enum NullValue { + // Null value. + NULL_VALUE = 0; +} + +// `ListValue` is a wrapper around a repeated field of values. +message ListValue { + // Repeated field of dynamically typed values. + repeated Value values = 1; +} diff --git a/src/google/protobuf/stubs/atomicops_internals_mips_gcc.h b/src/google/protobuf/stubs/atomicops_internals_mips_gcc.h index e3cd14cf..f5837c9e 100644 --- a/src/google/protobuf/stubs/atomicops_internals_mips_gcc.h +++ b/src/google/protobuf/stubs/atomicops_internals_mips_gcc.h @@ -65,7 +65,7 @@ inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, "2:\n" ".set pop\n" : "=&r" (prev), "=m" (*ptr), "=&r" (tmp) - : "Ir" (old_value), "r" (new_value), "m" (*ptr) + : "r" (old_value), "r" (new_value), "m" (*ptr) : "memory"); return prev; } @@ -197,7 +197,7 @@ inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, "2:\n" ".set pop\n" : "=&r" (prev), "=m" (*ptr), "=&r" (tmp) - : "Ir" (old_value), "r" (new_value), "m" (*ptr) + : "r" (old_value), "r" (new_value), "m" (*ptr) : "memory"); return prev; } diff --git a/src/google/protobuf/stubs/hash.h b/src/google/protobuf/stubs/hash.h index 5b6a073d..badcb6a5 100644 --- a/src/google/protobuf/stubs/hash.h +++ b/src/google/protobuf/stubs/hash.h @@ -37,13 +37,14 @@ #include <string.h> #include <google/protobuf/stubs/common.h> -#include "config.h" +#include <google/protobuf/stubs/pbconfig.h> -#if defined(HAVE_HASH_MAP) && defined(HAVE_HASH_SET) -#include HASH_MAP_H -#include HASH_SET_H +#if defined(GOOGLE_PROTOBUF_HAVE_HASH_MAP) && \ + defined(GOOGLE_PROTOBUF_HAVE_HASH_SET) +#include GOOGLE_PROTOBUF_HASH_MAP_H +#include GOOGLE_PROTOBUF_HASH_SET_H #else -#define MISSING_HASH +#define GOOGLE_PROTOBUF_MISSING_HASH #include <map> #include <set> #endif @@ -51,7 +52,7 @@ namespace google { namespace protobuf { -#ifdef MISSING_HASH +#ifdef GOOGLE_PROTOBUF_MISSING_HASH // This system doesn't have hash_map or hash_set. Emulate them using map and // set. @@ -105,7 +106,7 @@ class hash_set : public std::set<Key, HashFcn> { #elif defined(_MSC_VER) && !defined(_STLPORT_VERSION) template <typename Key> -struct hash : public HASH_NAMESPACE::hash_compare<Key> { +struct hash : public GOOGLE_PROTOBUF_HASH_NAMESPACE::hash_compare<Key> { }; // MSVC's hash_compare<const char*> hashes based on the string contents but @@ -119,13 +120,13 @@ class CstringLess { template <> struct hash<const char*> - : public HASH_NAMESPACE::hash_compare<const char*, CstringLess> { -}; + : public GOOGLE_PROTOBUF_HASH_NAMESPACE::hash_compare< + const char*, CstringLess> {}; template <typename Key, typename Data, typename HashFcn = hash<Key>, typename EqualKey = int > -class hash_map : public HASH_NAMESPACE::hash_map< +class hash_map : public GOOGLE_PROTOBUF_HASH_NAMESPACE::hash_map< Key, Data, HashFcn> { public: hash_map(int = 0) {} @@ -134,7 +135,7 @@ class hash_map : public HASH_NAMESPACE::hash_map< template <typename Key, typename HashFcn = hash<Key>, typename EqualKey = int > -class hash_set : public HASH_NAMESPACE::hash_set< +class hash_set : public GOOGLE_PROTOBUF_HASH_NAMESPACE::hash_set< Key, HashFcn> { public: hash_set(int = 0) {} @@ -143,7 +144,7 @@ class hash_set : public HASH_NAMESPACE::hash_set< #else template <typename Key> -struct hash : public HASH_NAMESPACE::hash<Key> { +struct hash : public GOOGLE_PROTOBUF_HASH_NAMESPACE::hash<Key> { }; template <typename Key> @@ -166,25 +167,26 @@ struct hash<const char*> { } }; -template <typename Key, typename Data, - typename HashFcn = hash<Key>, +template <typename Key, typename Data, typename HashFcn = hash<Key>, typename EqualKey = std::equal_to<Key> > -class hash_map : public HASH_NAMESPACE::HASH_MAP_CLASS< - Key, Data, HashFcn, EqualKey> { +class hash_map + : public GOOGLE_PROTOBUF_HASH_NAMESPACE::GOOGLE_PROTOBUF_HASH_MAP_CLASS< + Key, Data, HashFcn, EqualKey> { public: hash_map(int = 0) {} }; -template <typename Key, - typename HashFcn = hash<Key>, +template <typename Key, typename HashFcn = hash<Key>, typename EqualKey = std::equal_to<Key> > -class hash_set : public HASH_NAMESPACE::HASH_SET_CLASS< - Key, HashFcn, EqualKey> { +class hash_set + : public GOOGLE_PROTOBUF_HASH_NAMESPACE::GOOGLE_PROTOBUF_HASH_SET_CLASS< + Key, HashFcn, EqualKey> { public: hash_set(int = 0) {} }; -#endif +#undef GOOGLE_PROTOBUF_MISSING_HASH +#endif // !GOOGLE_PROTOBUF_MISSING_HASH template <> struct hash<string> { diff --git a/src/google/protobuf/stubs/singleton.h b/src/google/protobuf/stubs/singleton.h index e123e4fe..9301f549 100644 --- a/src/google/protobuf/stubs/singleton.h +++ b/src/google/protobuf/stubs/singleton.h @@ -44,6 +44,10 @@ class Singleton { GoogleOnceInit(&once_, &Singleton<T>::Init); return instance_; } + static void ShutDown() { + delete instance_; + instance_ = NULL; + } private: static void Init() { instance_ = new T(); @@ -56,7 +60,7 @@ template<typename T> ProtobufOnceType Singleton<T>::once_; template<typename T> -T* Singleton<T>::instance_; +T* Singleton<T>::instance_ = NULL; } // namespace internal } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/testdata/golden_message_proto3 b/src/google/protobuf/testdata/golden_message_proto3 Binary files differnew file mode 100644 index 00000000..934f36fa --- /dev/null +++ b/src/google/protobuf/testdata/golden_message_proto3 diff --git a/src/google/protobuf/timestamp.proto b/src/google/protobuf/timestamp.proto new file mode 100644 index 00000000..ac8e009c --- /dev/null +++ b/src/google/protobuf/timestamp.proto @@ -0,0 +1,103 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +syntax = "proto3"; + +package google.protobuf; + +option java_generate_equals_and_hash = true; +option java_multiple_files = true; +option java_outer_classname = "TimestampProto"; +option java_package = "com.google.protobuf"; + + +// A Timestamp represents a point in time independent of any time zone +// or calendar, represented as seconds and fractions of seconds at +// nanosecond resolution in UTC Epoch time. It is encoded using the +// Proleptic Gregorian Calendar which extends the Gregorian calendar +// backwards to year one. It is encoded assuming all minutes are 60 +// seconds long, i.e. leap seconds are "smeared" so that no leap second +// table is needed for interpretation. Range is from +// 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. +// By restricting to that range, we ensure that we can convert to +// and from RFC 3339 date strings. (See https://www.ietf.org/rfc/rfc3339.txt.) +// +// Example 1: compute Timestamp from POSIX `time()`. +// +// Timestamp timestamp; +// timestamp.set_seconds(time(NULL)); +// timestamp.set_nanos(0); +// +// Example 2: compute Timestamp from POSIX `gettimeofday()`. +// +// struct timeval tv; +// gettimeofday(&tv, NULL); +// +// Timestamp timestamp; +// timestamp.set_seconds(tv.tv_sec); +// timestamp.set_nanos(tv.tv_usec * 1000); +// +// Example 3: compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. +// +// FILETIME ft; +// GetSystemTimeAsFileTime(&ft); +// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; +// +// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z +// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. +// Timestamp timestamp; +// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); +// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); +// +// Example 4: compute Timestamp from Java `System.currentTimeMillis()`. +// +// long millis = System.currentTimeMillis(); +// +// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) +// .setNanos((int) ((millis % 1000) * 1000000)).build(); +// +// Example 5: compute Timestamp from Python `datetime.datetime`. +// +// now = datetime.datetime.utcnow() +// seconds = int(time.mktime(now.timetuple())) +// nanos = now.microsecond * 1000 +// timestamp = Timestamp(seconds=seconds, nanos=nanos) +// +message Timestamp { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + int64 seconds = 1; + + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + int32 nanos = 2; +} diff --git a/src/google/protobuf/unittest_proto3_arena.proto b/src/google/protobuf/unittest_proto3_arena.proto index 53f23d87..6d7bada8 100644 --- a/src/google/protobuf/unittest_proto3_arena.proto +++ b/src/google/protobuf/unittest_proto3_arena.proto @@ -47,9 +47,10 @@ message TestAllTypes { } enum NestedEnum { - FOO = 0; - BAR = 1; - BAZ = 2; + ZERO = 0; + FOO = 1; + BAR = 2; + BAZ = 3; NEG = -1; // Intentionally negative. } @@ -81,6 +82,11 @@ message TestAllTypes { optional NestedEnum optional_nested_enum = 21; optional ForeignEnum optional_foreign_enum = 22; + // Omitted (compared to unittest.proto) because proto2 enums are not allowed + // inside proto2 messages. + // + // optional protobuf_unittest_import.ImportEnum optional_import_enum = 23; + optional string optional_string_piece = 24 [ctype=STRING_PIECE]; optional string optional_cord = 25 [ctype=CORD]; @@ -118,6 +124,11 @@ message TestAllTypes { repeated NestedEnum repeated_nested_enum = 51; repeated ForeignEnum repeated_foreign_enum = 52; + // Omitted (compared to unittest.proto) because proto2 enums are not allowed + // inside proto2 messages. + // + // repeated protobuf_unittest_import.ImportEnum repeated_import_enum = 53; + repeated string repeated_string_piece = 54 [ctype=STRING_PIECE]; repeated string repeated_cord = 55 [ctype=CORD]; @@ -131,6 +142,31 @@ message TestAllTypes { } } +// Test messages for packed fields + +message TestPackedTypes { + repeated int32 packed_int32 = 90 [packed = true]; + repeated int64 packed_int64 = 91 [packed = true]; + repeated uint32 packed_uint32 = 92 [packed = true]; + repeated uint64 packed_uint64 = 93 [packed = true]; + repeated sint32 packed_sint32 = 94 [packed = true]; + repeated sint64 packed_sint64 = 95 [packed = true]; + repeated fixed32 packed_fixed32 = 96 [packed = true]; + repeated fixed64 packed_fixed64 = 97 [packed = true]; + repeated sfixed32 packed_sfixed32 = 98 [packed = true]; + repeated sfixed64 packed_sfixed64 = 99 [packed = true]; + repeated float packed_float = 100 [packed = true]; + repeated double packed_double = 101 [packed = true]; + repeated bool packed_bool = 102 [packed = true]; + repeated ForeignEnum packed_enum = 103 [packed = true]; +} + +// This proto includes a recusively nested message. +message NestedTestAllTypes { + NestedTestAllTypes child = 1; + TestAllTypes payload = 2; +} + // Define these after TestAllTypes to make sure the compiler can handle // that. message ForeignMessage { @@ -138,7 +174,13 @@ message ForeignMessage { } enum ForeignEnum { - FOREIGN_FOO = 0; - FOREIGN_BAR = 1; - FOREIGN_BAZ = 2; + FOREIGN_ZERO = 0; + FOREIGN_FOO = 4; + FOREIGN_BAR = 5; + FOREIGN_BAZ = 6; +} + +// TestEmptyMessage is used to test behavior of unknown fields. +message TestEmptyMessage { } + diff --git a/src/google/protobuf/wrappers.proto b/src/google/protobuf/wrappers.proto new file mode 100644 index 00000000..5164c24c --- /dev/null +++ b/src/google/protobuf/wrappers.proto @@ -0,0 +1,97 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Wrappers for primitive (non-message) types. These types are useful +// for embedding primitives in the `google.protobuf.Any` type and for places +// where we need to distinguish between the absence of a primitive +// typed field and its default value. + +syntax = "proto3"; + +package google.protobuf; + +option java_multiple_files = true; +option java_outer_classname = "WrappersProto"; +option java_package = "com.google.protobuf"; + + +// Wrapper message for double. +message DoubleValue { + // The double value. + double value = 1; +} + +// Wrapper message for float. +message FloatValue { + // The float value. + float value = 1; +} + +// Wrapper message for int64. +message Int64Value { + // The int64 value. + int64 value = 1; +} + +// Wrapper message for uint64. +message UInt64Value { + // The uint64 value. + uint64 value = 1; +} + +// Wrapper message for int32. +message Int32Value { + // The int32 value. + int32 value = 1; +} + +// Wrapper message for uint32. +message UInt32Value { + // The uint32 value. + uint32 value = 1; +} + +// Wrapper message for bool. +message BoolValue { + // The bool value. + bool value = 1; +} + +// Wrapper message for string. +message StringValue { + // The string value. + string value = 1; +} + +// Wrapper message for bytes. +message BytesValue { + // The bytes value. + bytes value = 1; +} diff --git a/vsprojects/extract_includes.bat b/vsprojects/extract_includes.bat index 587c1a8a..e9d762c6 100755 --- a/vsprojects/extract_includes.bat +++ b/vsprojects/extract_includes.bat @@ -6,7 +6,9 @@ md include\google\protobuf\io md include\google\protobuf\compiler md include\google\protobuf\compiler\cpp md include\google\protobuf\compiler\java +md include\google\protobuf\compiler\javanano md include\google\protobuf\compiler\python +md include\google\protobuf\compiler\ruby copy ..\src\google\protobuf\arena.h include\google\protobuf\arena.h copy ..\src\google\protobuf\arenastring.h include\google\protobuf\arenastring.h copy ..\src\google\protobuf\compiler\code_generator.h include\google\protobuf\compiler\code_generator.h @@ -14,10 +16,12 @@ copy ..\src\google\protobuf\compiler\command_line_interface.h include\google\pro copy ..\src\google\protobuf\compiler\cpp\cpp_generator.h include\google\protobuf\compiler\cpp\cpp_generator.h copy ..\src\google\protobuf\compiler\importer.h include\google\protobuf\compiler\importer.h copy ..\src\google\protobuf\compiler\java\java_generator.h include\google\protobuf\compiler\java\java_generator.h +copy ..\src\google\protobuf\compiler\javanano\javanano_generator.h include\google\protobuf\compiler\javanano\javanano_generator.h copy ..\src\google\protobuf\compiler\parser.h include\google\protobuf\compiler\parser.h copy ..\src\google\protobuf\compiler\plugin.h include\google\protobuf\compiler\plugin.h copy ..\src\google\protobuf\compiler\plugin.pb.h include\google\protobuf\compiler\plugin.pb.h copy ..\src\google\protobuf\compiler\python\python_generator.h include\google\protobuf\compiler\python\python_generator.h +copy ..\src\google\protobuf\compiler\ruby\ruby_generator.h include\google\protobuf\compiler\ruby\ruby_generator.h copy ..\src\google\protobuf\descriptor_database.h include\google\protobuf\descriptor_database.h copy ..\src\google\protobuf\descriptor.h include\google\protobuf\descriptor.h copy ..\src\google\protobuf\descriptor.pb.h include\google\protobuf\descriptor.pb.h @@ -68,6 +72,7 @@ copy ..\src\google\protobuf\stubs\fastmem.h include\google\protobuf\stubs\fastme copy ..\src\google\protobuf\stubs\once.h include\google\protobuf\stubs\once.h copy ..\src\google\protobuf\stubs\platform_macros.h include\google\protobuf\stubs\platform_macros.h copy ..\src\google\protobuf\stubs\singleton.h include\google\protobuf\stubs\singleton.h +copy ..\src\google\protobuf\stubs\hash.h include\google\protobuf\stubs\hash.h copy ..\src\google\protobuf\stubs\stl_util.h include\google\protobuf\stubs\stl_util.h copy ..\src\google\protobuf\stubs\template_util.h include\google\protobuf\stubs\template_util.h copy ..\src\google\protobuf\stubs\type_traits.h include\google\protobuf\stubs\type_traits.h @@ -76,3 +81,4 @@ copy ..\src\google\protobuf\unknown_field_set.h include\google\protobuf\unknown_ copy ..\src\google\protobuf\wire_format.h include\google\protobuf\wire_format.h copy ..\src\google\protobuf\wire_format_lite.h include\google\protobuf\wire_format_lite.h copy ..\src\google\protobuf\wire_format_lite_inl.h include\google\protobuf\wire_format_lite_inl.h +copy google\protobuf\stubs\pbconfig.h include\google\protobuf\stubs\pbconfig.h diff --git a/vsprojects/google/protobuf/stubs/pbconfig.h b/vsprojects/google/protobuf/stubs/pbconfig.h new file mode 100644 index 00000000..c49d26b0 --- /dev/null +++ b/vsprojects/google/protobuf/stubs/pbconfig.h @@ -0,0 +1,29 @@ +/* protobuf config.h for MSVC. On other platforms, this is generated + * automatically by autoheader / autoconf / configure. */ + +/* the location of <hash_map> */ +#define GOOGLE_PROTOBUF_HASH_MAP_H <hash_map> + +/* the namespace of hash_map/hash_set */ +// Apparently Microsoft decided to move hash_map *back* to the std namespace +// in MSVC 2010: +// http://blogs.msdn.com/vcblog/archive/2009/05/25/stl-breaking-changes-in-visual-studio-2010-beta-1.aspx +// TODO(kenton): Use unordered_map instead, which is available in MSVC 2010. +#if _MSC_VER < 1310 || _MSC_VER >= 1600 +#define GOOGLE_PROTOBUF_HASH_NAMESPACE std +#else +#define GOOGLE_PROTOBUF_HASH_NAMESPACE stdext +#endif + +/* the location of <hash_set> */ +#define GOOGLE_PROTOBUF_HASH_SET_H <hash_set> + +/* define if the compiler has hash_map */ +#define GOOGLE_PROTOBUF_HAVE_HASH_MAP 1 + +/* define if the compiler has hash_set */ +#define GOOGLE_PROTOBUF_HAVE_HASH_SET 1 + +/* define if you want to use zlib. See readme.txt for additional + * requirements. */ +// #define HAVE_ZLIB 1 diff --git a/vsprojects/libprotobuf-lite.vcproj b/vsprojects/libprotobuf-lite.vcproj index 09eca4d8..339d03fa 100644 --- a/vsprojects/libprotobuf-lite.vcproj +++ b/vsprojects/libprotobuf-lite.vcproj @@ -164,6 +164,10 @@ > </File> <File + RelativePath=".\google\protobuf\stubs\pbconfig.h" + > + </File> + <File RelativePath="..\src\google\protobuf\extension_set.h" > </File> diff --git a/vsprojects/libprotobuf.vcproj b/vsprojects/libprotobuf.vcproj index 095d3948..6712647d 100644 --- a/vsprojects/libprotobuf.vcproj +++ b/vsprojects/libprotobuf.vcproj @@ -164,6 +164,10 @@ > </File> <File + RelativePath=".\google\protobuf\stubs\pbconfig.h" + > + </File> + <File RelativePath="..\src\google\protobuf\descriptor.h" > </File> diff --git a/vsprojects/libprotoc.vcproj b/vsprojects/libprotoc.vcproj index 45a5936f..455c2cc2 100644 --- a/vsprojects/libprotoc.vcproj +++ b/vsprojects/libprotoc.vcproj @@ -1,518 +1,542 @@ <?xml version="1.0" encoding="UTF-8"?> <VisualStudioProject - ProjectType="Visual C++" - Version="9.00" - Name="libprotoc" - ProjectGUID="{B84FF31A-5F9A-46F8-AB22-DBFC9BECE3BE}" - Keyword="Win32Proj" - TargetFrameworkVersion="0" - > - <Platforms> - <Platform - Name="Win32" - /> - </Platforms> - <ToolFiles> - </ToolFiles> - <Configurations> - <Configuration - Name="Debug|Win32" - OutputDirectory="Debug" - IntermediateDirectory="$(OutDir)\$(ProjectName)" - ConfigurationType="4" - > - <Tool - Name="VCPreBuildEventTool" - /> - <Tool - Name="VCCustomBuildTool" - /> - <Tool - Name="VCXMLDataGeneratorTool" - /> - <Tool - Name="VCWebServiceProxyGeneratorTool" - /> - <Tool - Name="VCMIDLTool" - /> - <Tool - Name="VCCLCompilerTool" - AdditionalOptions="/wd4244 /wd4267 /wd4018 /wd4355 /wd4800 /wd4251 /wd4996 /wd4146 /wd4305" - Optimization="0" - AdditionalIncludeDirectories="../src;." - PreprocessorDefinitions="WIN32;_DEBUG;_WINDOWS;_USRDLL;LIBPROTOC_EXPORTS;" - MinimalRebuild="true" - BasicRuntimeChecks="3" - RuntimeLibrary="3" - UsePrecompiledHeader="0" - WarningLevel="3" - Detect64BitPortabilityProblems="true" - DebugInformationFormat="4" - /> - <Tool - Name="VCManagedResourceCompilerTool" - /> - <Tool - Name="VCResourceCompilerTool" - /> - <Tool - Name="VCPreLinkEventTool" - /> - <Tool - Name="VCLibrarianTool" - /> - <Tool - Name="VCALinkTool" - /> - <Tool - Name="VCXDCMakeTool" - /> - <Tool - Name="VCBscMakeTool" - /> - <Tool - Name="VCFxCopTool" - /> - <Tool - Name="VCPostBuildEventTool" - /> - </Configuration> - <Configuration - Name="Release|Win32" - OutputDirectory="Release" - IntermediateDirectory="$(OutDir)\$(ProjectName)" - ConfigurationType="4" - > - <Tool - Name="VCPreBuildEventTool" - /> - <Tool - Name="VCCustomBuildTool" - /> - <Tool - Name="VCXMLDataGeneratorTool" - /> - <Tool - Name="VCWebServiceProxyGeneratorTool" - /> - <Tool - Name="VCMIDLTool" - /> - <Tool - Name="VCCLCompilerTool" - AdditionalOptions="/wd4244 /wd4267 /wd4018 /wd4355 /wd4800 /wd4251 /wd4996 /wd4146 /wd4305" - AdditionalIncludeDirectories="../src;." - PreprocessorDefinitions="WIN32;NDEBUG;_WINDOWS;_USRDLL;LIBPROTOC_EXPORTS;" - RuntimeLibrary="2" - UsePrecompiledHeader="0" - WarningLevel="3" - Detect64BitPortabilityProblems="true" - DebugInformationFormat="3" - /> - <Tool - Name="VCManagedResourceCompilerTool" - /> - <Tool - Name="VCResourceCompilerTool" - /> - <Tool - Name="VCPreLinkEventTool" - /> - <Tool - Name="VCLibrarianTool" - /> - <Tool - Name="VCALinkTool" - /> - <Tool - Name="VCXDCMakeTool" - /> - <Tool - Name="VCBscMakeTool" - /> - <Tool - Name="VCFxCopTool" - /> - <Tool - Name="VCPostBuildEventTool" - /> - </Configuration> - </Configurations> - <References> - </References> - <Files> - <Filter - Name="Header Files" - Filter="h;hpp;hxx;hm;inl;inc;xsd" - UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}" - > - <File - RelativePath="..\src\google\protobuf\compiler\code_generator.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\command_line_interface.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\subprocess.h" - > - </File> + ProjectType="Visual C++" + Version="9.00" + Name="libprotoc" + ProjectGUID="{B84FF31A-5F9A-46F8-AB22-DBFC9BECE3BE}" + Keyword="Win32Proj" + TargetFrameworkVersion="0" + > + <Platforms> + <Platform + Name="Win32" + /> + </Platforms> + <ToolFiles> + </ToolFiles> + <Configurations> + <Configuration + Name="Debug|Win32" + OutputDirectory="Debug" + IntermediateDirectory="$(OutDir)\$(ProjectName)" + ConfigurationType="4" + > + <Tool + Name="VCPreBuildEventTool" + /> + <Tool + Name="VCCustomBuildTool" + /> + <Tool + Name="VCXMLDataGeneratorTool" + /> + <Tool + Name="VCWebServiceProxyGeneratorTool" + /> + <Tool + Name="VCMIDLTool" + /> + <Tool + Name="VCCLCompilerTool" + AdditionalOptions="/wd4244 /wd4267 /wd4018 /wd4355 /wd4800 /wd4251 /wd4996 /wd4146 /wd4305" + Optimization="0" + AdditionalIncludeDirectories="../src;." + PreprocessorDefinitions="WIN32;_DEBUG;_WINDOWS;_USRDLL;LIBPROTOC_EXPORTS;" + MinimalRebuild="true" + BasicRuntimeChecks="3" + RuntimeLibrary="3" + UsePrecompiledHeader="0" + WarningLevel="3" + Detect64BitPortabilityProblems="true" + DebugInformationFormat="4" + /> + <Tool + Name="VCManagedResourceCompilerTool" + /> + <Tool + Name="VCResourceCompilerTool" + /> + <Tool + Name="VCPreLinkEventTool" + /> + <Tool + Name="VCLibrarianTool" + /> + <Tool + Name="VCALinkTool" + /> + <Tool + Name="VCXDCMakeTool" + /> + <Tool + Name="VCBscMakeTool" + /> + <Tool + Name="VCFxCopTool" + /> + <Tool + Name="VCPostBuildEventTool" + /> + </Configuration> + <Configuration + Name="Release|Win32" + OutputDirectory="Release" + IntermediateDirectory="$(OutDir)\$(ProjectName)" + ConfigurationType="4" + > + <Tool + Name="VCPreBuildEventTool" + /> + <Tool + Name="VCCustomBuildTool" + /> + <Tool + Name="VCXMLDataGeneratorTool" + /> + <Tool + Name="VCWebServiceProxyGeneratorTool" + /> + <Tool + Name="VCMIDLTool" + /> + <Tool + Name="VCCLCompilerTool" + AdditionalOptions="/wd4244 /wd4267 /wd4018 /wd4355 /wd4800 /wd4251 /wd4996 /wd4146 /wd4305" + AdditionalIncludeDirectories="../src;." + PreprocessorDefinitions="WIN32;NDEBUG;_WINDOWS;_USRDLL;LIBPROTOC_EXPORTS;" + RuntimeLibrary="2" + UsePrecompiledHeader="0" + WarningLevel="3" + Detect64BitPortabilityProblems="true" + DebugInformationFormat="3" + /> + <Tool + Name="VCManagedResourceCompilerTool" + /> + <Tool + Name="VCResourceCompilerTool" + /> + <Tool + Name="VCPreLinkEventTool" + /> + <Tool + Name="VCLibrarianTool" + /> + <Tool + Name="VCALinkTool" + /> + <Tool + Name="VCXDCMakeTool" + /> + <Tool + Name="VCBscMakeTool" + /> + <Tool + Name="VCFxCopTool" + /> + <Tool + Name="VCPostBuildEventTool" + /> + </Configuration> + </Configurations> + <References> + </References> + <Files> + <Filter + Name="Header Files" + Filter="h;hpp;hxx;hm;inl;inc;xsd" + UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}" + > + <File + RelativePath="..\src\google\protobuf\compiler\code_generator.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\command_line_interface.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\subprocess.h" + > + </File> <File RelativePath="..\src\google\protobuf\compiler\zip_writer.h" > </File> - <File - RelativePath="..\src\google\protobuf\compiler\plugin.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\plugin.pb.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_enum.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_enum_field.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_extension.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_field.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_file.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_generator.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_helpers.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_message.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_message_field.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_options.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_primitive_field.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_service.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_string_field.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_context.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_enum.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_enum_field.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_extension.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_field.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_file.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_generator.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_generator_factory.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_helpers.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_lazy_message_field.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_message.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_message_field.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_name_resolver.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_primitive_field.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_service.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_shared_code_generator.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_string_field.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_doc_comment.h" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_doc_comment.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\python\python_generator.h" - > - </File> - </Filter> - <Filter - Name="Resource Files" - Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx" - UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}" - > - </Filter> - <Filter - Name="Source Files" - Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx" - UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}" - > - <File - RelativePath="..\src\google\protobuf\compiler\code_generator.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\command_line_interface.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_enum.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_enum_field.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_extension.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_field.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_file.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_generator.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_helpers.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_map_field.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_message.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_message_field.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_primitive_field.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_service.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\cpp\cpp_string_field.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_context.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_doc_comment.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_enum.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_enum_field.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_extension.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_field.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_file.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_generator.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_generator_factory.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_helpers.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_lazy_message_field.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_map_field.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_message.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_message_field.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_name_resolver.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_primitive_field.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_service.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_shared_code_generator.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\java\java_string_field.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\javanano\javanano_enum.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\javanano\javanano_enum_field.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\javanano\javanano_extension.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\javanano\javanano_field.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\javanano\javanano_file.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\javanano\javanano_generator.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\javanano\javanano_helpers.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\javanano\javanano_message.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\javanano\javanano_message_field.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\javanano\javanano_primitive_field.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\plugin.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\plugin.pb.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\python\python_generator.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\subprocess.cc" - > - </File> - <File - RelativePath="..\src\google\protobuf\compiler\zip_writer.cc" - > - </File> - </Filter> - </Files> - <Globals> - </Globals> + <File + RelativePath="..\src\google\protobuf\compiler\plugin.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\plugin.pb.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_enum.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_enum_field.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_extension.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_field.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_file.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_generator.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_helpers.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_message.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_message_field.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_options.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_primitive_field.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_service.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_string_field.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_context.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_enum.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_enum_field.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_extension.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_field.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_file.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_generator.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_generator_factory.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_helpers.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_lazy_message_field.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_message.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_message_field.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_name_resolver.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_primitive_field.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_service.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_shared_code_generator.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_string_field.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_doc_comment.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_doc_comment.cc" + > + </File> + <File RelativePath="..\src\google\protobuf\compiler\javanano\javanano_enum_field.h"></File> + <File RelativePath="..\src\google\protobuf\compiler\javanano\javanano_enum.h"></File> + <File RelativePath="..\src\google\protobuf\compiler\javanano\javanano_extension.h"></File> + <File RelativePath="..\src\google\protobuf\compiler\javanano\javanano_field.h"></File> + <File RelativePath="..\src\google\protobuf\compiler\javanano\javanano_file.h"></File> + <File RelativePath="..\src\google\protobuf\compiler\javanano\javanano_generator.h"></File> + <File RelativePath="..\src\google\protobuf\compiler\javanano\javanano_helpers.h"></File> + <File RelativePath="..\src\google\protobuf\compiler\javanano\javanano_map_field.h"></File> + <File RelativePath="..\src\google\protobuf\compiler\javanano\javanano_message_field.h"></File> + <File RelativePath="..\src\google\protobuf\compiler\javanano\javanano_message.h"></File> + <File RelativePath="..\src\google\protobuf\compiler\javanano\javanano_params.h"></File> + <File RelativePath="..\src\google\protobuf\compiler\javanano\javanano_primitive_field.h"></File> + <File + RelativePath="..\src\google\protobuf\compiler\python\python_generator.h" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\ruby\ruby_generator.h" + > + </File> + </Filter> + <Filter + Name="Resource Files" + Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx" + UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}" + > + </Filter> + <Filter + Name="Source Files" + Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx" + UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}" + > + <File + RelativePath="..\src\google\protobuf\compiler\code_generator.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\command_line_interface.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_enum.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_enum_field.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_extension.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_field.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_file.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_generator.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_helpers.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_map_field.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_message.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_message_field.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_primitive_field.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_service.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\cpp\cpp_string_field.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\ruby\ruby_generator.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_context.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_doc_comment.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_enum.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_enum_field.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_extension.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_field.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_file.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_generator.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_generator_factory.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_helpers.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_lazy_message_field.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_map_field.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_message.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_message_field.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_name_resolver.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_primitive_field.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_service.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_shared_code_generator.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\java\java_string_field.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\javanano\javanano_enum.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\javanano\javanano_enum_field.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\javanano\javanano_extension.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\javanano\javanano_field.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\javanano\javanano_file.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\javanano\javanano_generator.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\javanano\javanano_helpers.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\javanano\javanano_map_field.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\javanano\javanano_message.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\javanano\javanano_message_field.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\javanano\javanano_primitive_field.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\plugin.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\plugin.pb.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\python\python_generator.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\subprocess.cc" + > + </File> + <File + RelativePath="..\src\google\protobuf\compiler\zip_writer.cc" + > + </File> + </Filter> + </Files> + <Globals> + </Globals> </VisualStudioProject> |