From b55a20fa2c669b181f47ea9219b8e74d1263da19 Mon Sep 17 00:00:00 2001 From: "xiaofeng@google.com" Date: Sat, 22 Sep 2012 02:40:50 +0000 Subject: Down-integrate from internal branch --- .../java/com/google/protobuf/AbstractMessage.java | 336 ++++++-- .../com/google/protobuf/AbstractMessageLite.java | 26 +- .../java/com/google/protobuf/AbstractParser.java | 261 ++++++ .../com/google/protobuf/BoundedByteString.java | 163 ++++ .../main/java/com/google/protobuf/ByteString.java | 877 +++++++++++++++---- .../java/com/google/protobuf/CodedInputStream.java | 41 +- .../com/google/protobuf/CodedOutputStream.java | 34 +- .../main/java/com/google/protobuf/Descriptors.java | 157 +++- .../java/com/google/protobuf/DynamicMessage.java | 68 +- .../com/google/protobuf/ExtensionRegistryLite.java | 18 +- .../main/java/com/google/protobuf/FieldSet.java | 137 ++- .../java/com/google/protobuf/GeneratedMessage.java | 179 +++- .../com/google/protobuf/GeneratedMessageLite.java | 280 +++--- .../main/java/com/google/protobuf/Internal.java | 95 +-- .../protobuf/InvalidProtocolBufferException.java | 21 + .../main/java/com/google/protobuf/LazyField.java | 216 +++++ .../com/google/protobuf/LazyStringArrayList.java | 30 +- .../java/com/google/protobuf/LazyStringList.java | 17 +- .../com/google/protobuf/LiteralByteString.java | 349 ++++++++ .../src/main/java/com/google/protobuf/Message.java | 30 +- .../main/java/com/google/protobuf/MessageLite.java | 40 +- .../com/google/protobuf/MessageLiteOrBuilder.java | 2 + .../java/com/google/protobuf/MessageOrBuilder.java | 23 +- java/src/main/java/com/google/protobuf/Parser.java | 259 ++++++ .../com/google/protobuf/RepeatedFieldBuilder.java | 14 +- .../java/com/google/protobuf/RopeByteString.java | 945 +++++++++++++++++++++ .../com/google/protobuf/SingleFieldBuilder.java | 12 +- .../java/com/google/protobuf/SmallSortedMap.java | 10 +- .../main/java/com/google/protobuf/TextFormat.java | 121 ++- .../java/com/google/protobuf/UnknownFieldSet.java | 29 +- .../protobuf/UnmodifiableLazyStringList.java | 7 + java/src/main/java/com/google/protobuf/Utf8.java | 349 ++++++++ .../main/java/com/google/protobuf/WireFormat.java | 2 +- .../com/google/protobuf/AbstractMessageTest.java | 51 +- .../com/google/protobuf/BoundedByteStringTest.java | 68 ++ .../java/com/google/protobuf/ByteStringTest.java | 692 +++++++++++++++ .../com/google/protobuf/CodedOutputStreamTest.java | 17 +- .../java/com/google/protobuf/DescriptorsTest.java | 192 ++++- .../com/google/protobuf/DynamicMessageTest.java | 72 +- .../com/google/protobuf/GeneratedMessageTest.java | 168 +++- .../java/com/google/protobuf/IsValidUtf8Test.java | 180 ++++ .../com/google/protobuf/IsValidUtf8TestUtil.java | 421 +++++++++ .../google/protobuf/LazyStringArrayListTest.java | 44 + .../google/protobuf/LazyStringEndToEndTest.java | 28 + .../com/google/protobuf/LiteralByteStringTest.java | 396 +++++++++ .../test/java/com/google/protobuf/MessageTest.java | 40 + .../test/java/com/google/protobuf/ParserTest.java | 375 ++++++++ .../protobuf/RopeByteStringSubstringTest.java | 97 +++ .../com/google/protobuf/RopeByteStringTest.java | 115 +++ .../com/google/protobuf/TestBadIdentifiers.java | 18 +- .../test/java/com/google/protobuf/TestUtil.java | 212 ++++- .../java/com/google/protobuf/TextFormatTest.java | 34 + .../java/com/google/protobuf/WireFormatTest.java | 157 +++- .../com/google/protobuf/test_bad_identifiers.proto | 18 + 54 files changed, 7846 insertions(+), 697 deletions(-) create mode 100644 java/src/main/java/com/google/protobuf/AbstractParser.java create mode 100644 java/src/main/java/com/google/protobuf/BoundedByteString.java create mode 100644 java/src/main/java/com/google/protobuf/LazyField.java create mode 100644 java/src/main/java/com/google/protobuf/LiteralByteString.java create mode 100644 java/src/main/java/com/google/protobuf/Parser.java create mode 100644 java/src/main/java/com/google/protobuf/RopeByteString.java create mode 100644 java/src/main/java/com/google/protobuf/Utf8.java create mode 100644 java/src/test/java/com/google/protobuf/BoundedByteStringTest.java create mode 100644 java/src/test/java/com/google/protobuf/ByteStringTest.java create mode 100644 java/src/test/java/com/google/protobuf/IsValidUtf8Test.java create mode 100644 java/src/test/java/com/google/protobuf/IsValidUtf8TestUtil.java create mode 100644 java/src/test/java/com/google/protobuf/LiteralByteStringTest.java create mode 100644 java/src/test/java/com/google/protobuf/ParserTest.java create mode 100644 java/src/test/java/com/google/protobuf/RopeByteStringSubstringTest.java create mode 100644 java/src/test/java/com/google/protobuf/RopeByteStringTest.java (limited to 'java/src') diff --git a/java/src/main/java/com/google/protobuf/AbstractMessage.java b/java/src/main/java/com/google/protobuf/AbstractMessage.java index b9d83016..f4d115de 100644 --- a/java/src/main/java/com/google/protobuf/AbstractMessage.java +++ b/java/src/main/java/com/google/protobuf/AbstractMessage.java @@ -32,6 +32,7 @@ package com.google.protobuf; import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Descriptors.FieldDescriptor; +import com.google.protobuf.GeneratedMessage.ExtendableBuilder; import com.google.protobuf.Internal.EnumLite; import java.io.IOException; @@ -81,6 +82,25 @@ public abstract class AbstractMessage extends AbstractMessageLite return true; } + public List findInitializationErrors() { + return Builder.findMissingFields(this); + } + + public String getInitializationErrorString() { + return delimitWithCommas(findInitializationErrors()); + } + + private static String delimitWithCommas(List parts) { + StringBuilder result = new StringBuilder(); + for (String part : parts) { + if (result.length() > 0) { + result.append(", "); + } + result.append(part); + } + return result.toString(); + } + @Override public final String toString() { return TextFormat.printToString(this); @@ -208,6 +228,15 @@ public abstract class AbstractMessage extends AbstractMessageLite return b ? 1231 : 1237; } + /** + * Package private helper method for AbstractParser to create + * UninitializedMessageException with missing field information. + */ + @Override + UninitializedMessageException newUninitializedMessageException() { + return Builder.newUninitializedMessageException(this); + } + /** * Helper method for implementing {@link Message#hashCode()}. *

@@ -251,6 +280,14 @@ public abstract class AbstractMessage extends AbstractMessageLite return (BuilderType) this; } + public List findInitializationErrors() { + return findMissingFields(this); + } + + public String getInitializationErrorString() { + return delimitWithCommas(findInitializationErrors()); + } + public BuilderType mergeFrom(final Message other) { if (other.getDescriptorForType() != getDescriptorForType()) { throw new IllegalArgumentException( @@ -314,7 +351,7 @@ public abstract class AbstractMessage extends AbstractMessageLite } if (!mergeFieldFrom(input, unknownFields, extensionRegistry, - this, tag)) { + getDescriptorForType(), this, null, tag)) { // end group tag break; } @@ -323,25 +360,93 @@ public abstract class AbstractMessage extends AbstractMessageLite return (BuilderType) this; } + /** helper method to handle {@code builder} and {@code extensions}. */ + private static void addRepeatedField( + Message.Builder builder, + FieldSet extensions, + FieldDescriptor field, + Object value) { + if (builder != null) { + builder.addRepeatedField(field, value); + } else { + extensions.addRepeatedField(field, value); + } + } + + /** helper method to handle {@code builder} and {@code extensions}. */ + private static void setField( + Message.Builder builder, + FieldSet extensions, + FieldDescriptor field, + Object value) { + if (builder != null) { + builder.setField(field, value); + } else { + extensions.setField(field, value); + } + } + + /** helper method to handle {@code builder} and {@code extensions}. */ + private static boolean hasOriginalMessage( + Message.Builder builder, + FieldSet extensions, + FieldDescriptor field) { + if (builder != null) { + return builder.hasField(field); + } else { + return extensions.hasField(field); + } + } + + /** helper method to handle {@code builder} and {@code extensions}. */ + private static Message getOriginalMessage( + Message.Builder builder, + FieldSet extensions, + FieldDescriptor field) { + if (builder != null) { + return (Message) builder.getField(field); + } else { + return (Message) extensions.getField(field); + } + } + + /** helper method to handle {@code builder} and {@code extensions}. */ + private static void mergeOriginalMessage( + Message.Builder builder, + FieldSet extensions, + FieldDescriptor field, + Message.Builder subBuilder) { + Message originalMessage = getOriginalMessage(builder, extensions, field); + if (originalMessage != null) { + subBuilder.mergeFrom(originalMessage); + } + } + /** - * Like {@link #mergeFrom(CodedInputStream, UnknownFieldSet.Builder, - * ExtensionRegistryLite, Message.Builder)}, but parses a single field. + * Like {@link #mergeFrom(CodedInputStream, ExtensionRegistryLite)}, but + * parses a single field. + * + * When {@code builder} is not null, the method will parse and merge the + * field into {@code builder}. Otherwise, it will try to parse the field + * into {@code extensions}, when it's called by the parsing constructor in + * generated classes. + * * Package-private because it is used by GeneratedMessage.ExtendableMessage. * @param tag The tag, which should have already been read. * @return {@code true} unless the tag is an end-group tag. */ static boolean mergeFieldFrom( - final CodedInputStream input, - final UnknownFieldSet.Builder unknownFields, - final ExtensionRegistryLite extensionRegistry, - final Message.Builder builder, - final int tag) throws IOException { - final Descriptor type = builder.getDescriptorForType(); - + CodedInputStream input, + UnknownFieldSet.Builder unknownFields, + ExtensionRegistryLite extensionRegistry, + Descriptor type, + Message.Builder builder, + FieldSet extensions, + int tag) throws IOException { if (type.getOptions().getMessageSetWireFormat() && tag == WireFormat.MESSAGE_SET_ITEM_TAG) { mergeMessageSetExtensionFromCodedStream( - input, unknownFields, extensionRegistry, builder); + input, unknownFields, extensionRegistry, type, builder, extensions); return true; } @@ -376,8 +481,10 @@ public abstract class AbstractMessage extends AbstractMessageLite } else { field = null; } - } else { + } else if (builder != null) { field = type.findFieldByNumber(fieldNumber); + } else { + field = null; } boolean unknown = false; @@ -413,13 +520,13 @@ public abstract class AbstractMessage extends AbstractMessageLite // enum, drop it (don't even add it to unknownFields). return true; } - builder.addRepeatedField(field, value); + addRepeatedField(builder, extensions, field, value); } } else { while (input.getBytesUntilLimit() > 0) { final Object value = FieldSet.readPrimitiveField(input, field.getLiteType()); - builder.addRepeatedField(field, value); + addRepeatedField(builder, extensions, field, value); } } input.popLimit(limit); @@ -434,10 +541,10 @@ public abstract class AbstractMessage extends AbstractMessageLite subBuilder = builder.newBuilderForField(field); } if (!field.isRepeated()) { - subBuilder.mergeFrom((Message) builder.getField(field)); + mergeOriginalMessage(builder, extensions, field, subBuilder); } input.readGroup(field.getNumber(), subBuilder, extensionRegistry); - value = subBuilder.build(); + value = subBuilder.buildPartial(); break; } case MESSAGE: { @@ -448,10 +555,10 @@ public abstract class AbstractMessage extends AbstractMessageLite subBuilder = builder.newBuilderForField(field); } if (!field.isRepeated()) { - subBuilder.mergeFrom((Message) builder.getField(field)); + mergeOriginalMessage(builder, extensions, field, subBuilder); } input.readMessage(subBuilder, extensionRegistry); - value = subBuilder.build(); + value = subBuilder.buildPartial(); break; } case ENUM: @@ -470,22 +577,28 @@ public abstract class AbstractMessage extends AbstractMessageLite } if (field.isRepeated()) { - builder.addRepeatedField(field, value); + addRepeatedField(builder, extensions, field, value); } else { - builder.setField(field, value); + setField(builder, extensions, field, value); } } return true; } - /** Called by {@code #mergeFieldFrom()} to parse a MessageSet extension. */ + /** + * Called by {@code #mergeFieldFrom()} to parse a MessageSet extension. + * If {@code builder} is not null, this method will merge MessageSet into + * the builder. Otherwise, it will merge the MessageSet into {@code + * extensions}. + */ private static void mergeMessageSetExtensionFromCodedStream( - final CodedInputStream input, - final UnknownFieldSet.Builder unknownFields, - final ExtensionRegistryLite extensionRegistry, - final Message.Builder builder) throws IOException { - final Descriptor type = builder.getDescriptorForType(); + CodedInputStream input, + UnknownFieldSet.Builder unknownFields, + ExtensionRegistryLite extensionRegistry, + Descriptor type, + Message.Builder builder, + FieldSet extensions) throws IOException { // The wire format for MessageSet is: // message MessageSet { @@ -504,10 +617,11 @@ public abstract class AbstractMessage extends AbstractMessageLite // should be prepared to accept them. int typeId = 0; - ByteString rawBytes = null; // If we encounter "message" before "typeId" - Message.Builder subBuilder = null; - FieldDescriptor field = null; + ByteString rawBytes = null; // If we encounter "message" before "typeId" + ExtensionRegistry.ExtensionInfo extension = null; + // Read bytes from input, if we get it's type first then parse it eagerly, + // otherwise we store the raw bytes in a local variable. while (true) { final int tag = input.readTag(); if (tag == 0) { @@ -516,75 +630,121 @@ public abstract class AbstractMessage extends AbstractMessageLite if (tag == WireFormat.MESSAGE_SET_TYPE_ID_TAG) { typeId = input.readUInt32(); - // Zero is not a valid type ID. if (typeId != 0) { - final ExtensionRegistry.ExtensionInfo extension; - // extensionRegistry may be either ExtensionRegistry or - // ExtensionRegistryLite. Since the type we are parsing is a full + // ExtensionRegistryLite. Since the type we are parsing is a full // message, only a full ExtensionRegistry could possibly contain - // extensions of it. Otherwise we will treat the registry as if it + // extensions of it. Otherwise we will treat the registry as if it // were empty. if (extensionRegistry instanceof ExtensionRegistry) { extension = ((ExtensionRegistry) extensionRegistry) .findExtensionByNumber(type, typeId); - } else { - extension = null; - } - - if (extension != null) { - field = extension.descriptor; - subBuilder = extension.defaultInstance.newBuilderForType(); - final Message originalMessage = (Message)builder.getField(field); - if (originalMessage != null) { - subBuilder.mergeFrom(originalMessage); - } - if (rawBytes != null) { - // We already encountered the message. Parse it now. - subBuilder.mergeFrom( - CodedInputStream.newInstance(rawBytes.newInput())); - rawBytes = null; - } - } else { - // Unknown extension number. If we already saw data, put it - // in rawBytes. - if (rawBytes != null) { - unknownFields.mergeField(typeId, - UnknownFieldSet.Field.newBuilder() - .addLengthDelimited(rawBytes) - .build()); - rawBytes = null; - } } } + } else if (tag == WireFormat.MESSAGE_SET_MESSAGE_TAG) { - if (typeId == 0) { - // We haven't seen a type ID yet, so we have to store the raw bytes - // for now. - rawBytes = input.readBytes(); - } else if (subBuilder == null) { - // We don't know how to parse this. Ignore it. - unknownFields.mergeField(typeId, - UnknownFieldSet.Field.newBuilder() - .addLengthDelimited(input.readBytes()) - .build()); - } else { - // We already know the type, so we can parse directly from the input - // with no copying. Hooray! - input.readMessage(subBuilder, extensionRegistry); + if (typeId != 0) { + if (extension != null && ExtensionRegistryLite.isEagerlyParseMessageSets()) { + // We already know the type, so we can parse directly from the + // input with no copying. Hooray! + eagerlyMergeMessageSetExtension( + input, extension, extensionRegistry, builder, extensions); + rawBytes = null; + continue; + } } - } else { - // Unknown tag. Skip it. + // We haven't seen a type ID yet or we want parse message lazily. + rawBytes = input.readBytes(); + + } else { // Unknown tag. Skip it. if (!input.skipField(tag)) { - break; // end of group + break; // End of group } } } - input.checkLastTagWas(WireFormat.MESSAGE_SET_ITEM_END_TAG); - if (subBuilder != null) { - builder.setField(field, subBuilder.build()); + // Process the raw bytes. + if (rawBytes != null && typeId != 0) { // Zero is not a valid type ID. + if (extension != null) { // We known the type + mergeMessageSetExtensionFromBytes( + rawBytes, extension, extensionRegistry, builder, extensions); + } else { // We don't know how to parse this. Ignore it. + if (rawBytes != null) { + unknownFields.mergeField(typeId, UnknownFieldSet.Field.newBuilder() + .addLengthDelimited(rawBytes).build()); + } + } + } + } + + private static void eagerlyMergeMessageSetExtension( + CodedInputStream input, + ExtensionRegistry.ExtensionInfo extension, + ExtensionRegistryLite extensionRegistry, + Message.Builder builder, + FieldSet extensions) throws IOException { + + FieldDescriptor field = extension.descriptor; + Message value = null; + if (hasOriginalMessage(builder, extensions, field)) { + Message originalMessage = + getOriginalMessage(builder, extensions, field); + Message.Builder subBuilder = originalMessage.toBuilder(); + input.readMessage(subBuilder, extensionRegistry); + value = subBuilder.buildPartial(); + } else { + value = input.readMessage(extension.defaultInstance.getParserForType(), + extensionRegistry); + } + + if (builder != null) { + builder.setField(field, value); + } else { + extensions.setField(field, value); + } + } + + private static void mergeMessageSetExtensionFromBytes( + ByteString rawBytes, + ExtensionRegistry.ExtensionInfo extension, + ExtensionRegistryLite extensionRegistry, + Message.Builder builder, + FieldSet extensions) throws IOException { + + FieldDescriptor field = extension.descriptor; + boolean hasOriginalValue = hasOriginalMessage(builder, extensions, field); + + if (hasOriginalValue || ExtensionRegistryLite.isEagerlyParseMessageSets()) { + // If the field already exists, we just parse the field. + Message value = null; + if (hasOriginalValue) { + Message originalMessage = + getOriginalMessage(builder, extensions, field); + Message.Builder subBuilder= originalMessage.toBuilder(); + subBuilder.mergeFrom(rawBytes, extensionRegistry); + value = subBuilder.buildPartial(); + } else { + value = extension.defaultInstance.getParserForType() + .parsePartialFrom(rawBytes, extensionRegistry); + } + setField(builder, extensions, field, value); + } else { + // Use LazyField to load MessageSet lazily. + LazyField lazyField = new LazyField( + extension.defaultInstance, extensionRegistry, rawBytes); + if (builder != null) { + // TODO(xiangl): it looks like this method can only be invoked by + // ExtendableBuilder, but I'm not sure. So I double check the type of + // builder here. It may be useless and need more investigation. + if (builder instanceof ExtendableBuilder) { + builder.setField(field, lazyField); + } else { + builder.setField(field, lazyField.getValue()); + } + } else { + extensions.setField(field, lazyField); + } } } @@ -596,6 +756,11 @@ public abstract class AbstractMessage extends AbstractMessageLite return (BuilderType) this; } + public Message.Builder getFieldBuilder(final FieldDescriptor field) { + throw new UnsupportedOperationException( + "getFieldBuilder() called on an unsupported message type."); + } + /** * Construct an UninitializedMessageException reporting missing fields in * the given message. @@ -609,14 +774,15 @@ public abstract class AbstractMessage extends AbstractMessageLite * Populates {@code this.missingFields} with the full "path" of each * missing required field in the given message. */ - private static List findMissingFields(final Message message) { + private static List findMissingFields( + final MessageOrBuilder message) { final List results = new ArrayList(); findMissingFields(message, "", results); return results; } /** Recursive helper implementing {@link #findMissingFields(Message)}. */ - private static void findMissingFields(final Message message, + private static void findMissingFields(final MessageOrBuilder message, final String prefix, final List results) { for (final FieldDescriptor field : @@ -635,13 +801,13 @@ public abstract class AbstractMessage extends AbstractMessageLite if (field.isRepeated()) { int i = 0; for (final Object element : (List) value) { - findMissingFields((Message) element, + findMissingFields((MessageOrBuilder) element, subMessagePrefix(prefix, field, i++), results); } } else { if (message.hasField(field)) { - findMissingFields((Message) value, + findMissingFields((MessageOrBuilder) value, subMessagePrefix(prefix, field, -1), results); } diff --git a/java/src/main/java/com/google/protobuf/AbstractMessageLite.java b/java/src/main/java/com/google/protobuf/AbstractMessageLite.java index 77b27370..9926f3db 100644 --- a/java/src/main/java/com/google/protobuf/AbstractMessageLite.java +++ b/java/src/main/java/com/google/protobuf/AbstractMessageLite.java @@ -91,6 +91,14 @@ public abstract class AbstractMessageLite implements MessageLite { codedOutput.flush(); } + /** + * Package private helper method for AbstractParser to create + * UninitializedMessageException. + */ + UninitializedMessageException newUninitializedMessageException() { + return new UninitializedMessageException(this); + } + /** * A partial implementation of the {@link Message.Builder} interface which * implements as many methods of that interface as possible in terms of @@ -307,10 +315,12 @@ public abstract class AbstractMessageLite implements MessageLite { */ protected static void addAll(final Iterable values, final Collection list) { - for (final T value : values) { - if (value == null) { - throw new NullPointerException(); - } + if (values instanceof LazyStringList) { + // For StringOrByteStringLists, check the underlying elements to avoid + // forcing conversions of ByteStrings to Strings. + checkForNullValues(((LazyStringList) values).getUnderlyingElements()); + } else { + checkForNullValues(values); } if (values instanceof Collection) { final Collection collection = (Collection) values; @@ -321,5 +331,13 @@ public abstract class AbstractMessageLite implements MessageLite { } } } + + private static void checkForNullValues(final Iterable values) { + for (final Object value : values) { + if (value == null) { + throw new NullPointerException(); + } + } + } } } diff --git a/java/src/main/java/com/google/protobuf/AbstractParser.java b/java/src/main/java/com/google/protobuf/AbstractParser.java new file mode 100644 index 00000000..9bd9d397 --- /dev/null +++ b/java/src/main/java/com/google/protobuf/AbstractParser.java @@ -0,0 +1,261 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// http://code.google.com/p/protobuf/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import com.google.protobuf.AbstractMessageLite.Builder.LimitedInputStream; + +import java.io.IOException; +import java.io.InputStream; + +/** + * A partial implementation of the {@link Parser} interface which implements + * as many methods of that interface as possible in terms of other methods. + * + * Note: This class implements all the convenience methods in the + * {@link Parser} interface. See {@link Parser} for related javadocs. + * Subclasses need to implement + * {@link Parser#parsePartialFrom(CodedInputStream, ExtensionRegistryLite)} + * + * @author liujisi@google.com (Pherl Liu) + */ +public abstract class AbstractParser + implements Parser { + /** + * Creates an UninitializedMessageException for MessageType. + */ + private UninitializedMessageException + newUninitializedMessageException(MessageType message) { + if (message instanceof AbstractMessageLite) { + return ((AbstractMessageLite) message).newUninitializedMessageException(); + } + return new UninitializedMessageException(message); + } + + /** + * Helper method to check if message is initialized. + * + * @throws InvalidProtocolBufferException if it is not initialized. + * @return The message to check. + */ + private MessageType checkMessageInitialized(MessageType message) + throws InvalidProtocolBufferException { + if (message != null && !message.isInitialized()) { + throw newUninitializedMessageException(message) + .asInvalidProtocolBufferException() + .setUnfinishedMessage(message); + } + return message; + } + + private static final ExtensionRegistryLite EMPTY_REGISTRY + = ExtensionRegistryLite.getEmptyRegistry(); + + public MessageType parsePartialFrom(CodedInputStream input) + throws InvalidProtocolBufferException { + return parsePartialFrom(input, EMPTY_REGISTRY); + } + + public MessageType parseFrom(CodedInputStream input, + ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + return checkMessageInitialized( + parsePartialFrom(input, extensionRegistry)); + } + + public MessageType parseFrom(CodedInputStream input) + throws InvalidProtocolBufferException { + return parseFrom(input, EMPTY_REGISTRY); + } + + public MessageType parsePartialFrom(ByteString data, + ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + MessageType message; + try { + CodedInputStream input = data.newCodedInput(); + message = parsePartialFrom(input, extensionRegistry); + try { + input.checkLastTagWas(0); + } catch (InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(message); + } + return message; + } catch (InvalidProtocolBufferException e) { + throw e; + } catch (IOException e) { + throw new RuntimeException( + "Reading from a ByteString threw an IOException (should " + + "never happen).", e); + } + } + + public MessageType parsePartialFrom(ByteString data) + throws InvalidProtocolBufferException { + return parsePartialFrom(data, EMPTY_REGISTRY); + } + + public MessageType parseFrom(ByteString data, + ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + return checkMessageInitialized(parsePartialFrom(data, extensionRegistry)); + } + + public MessageType parseFrom(ByteString data) + throws InvalidProtocolBufferException { + return parseFrom(data, EMPTY_REGISTRY); + } + + public MessageType parsePartialFrom(byte[] data, int off, int len, + ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + try { + CodedInputStream input = CodedInputStream.newInstance(data, off, len); + MessageType message = parsePartialFrom(input, extensionRegistry); + try { + input.checkLastTagWas(0); + } catch (InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(message); + } + return message; + } catch (InvalidProtocolBufferException e) { + throw e; + } catch (IOException e) { + throw new RuntimeException( + "Reading from a byte array threw an IOException (should " + + "never happen).", e); + } + } + + public MessageType parsePartialFrom(byte[] data, int off, int len) + throws InvalidProtocolBufferException { + return parsePartialFrom(data, off, len, EMPTY_REGISTRY); + } + + public MessageType parsePartialFrom(byte[] data, + ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + return parsePartialFrom(data, 0, data.length, extensionRegistry); + } + + public MessageType parsePartialFrom(byte[] data) + throws InvalidProtocolBufferException { + return parsePartialFrom(data, 0, data.length, EMPTY_REGISTRY); + } + + public MessageType parseFrom(byte[] data, int off, int len, + ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + return checkMessageInitialized( + parsePartialFrom(data, off, len, extensionRegistry)); + } + + public MessageType parseFrom(byte[] data, int off, int len) + throws InvalidProtocolBufferException { + return parseFrom(data, off, len, EMPTY_REGISTRY); + } + + public MessageType parseFrom(byte[] data, + ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + return parseFrom(data, 0, data.length, extensionRegistry); + } + + public MessageType parseFrom(byte[] data) + throws InvalidProtocolBufferException { + return parseFrom(data, EMPTY_REGISTRY); + } + + public MessageType parsePartialFrom(InputStream input, + ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + CodedInputStream codedInput = CodedInputStream.newInstance(input); + MessageType message = parsePartialFrom(codedInput, extensionRegistry); + try { + codedInput.checkLastTagWas(0); + } catch (InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(message); + } + return message; + } + + public MessageType parsePartialFrom(InputStream input) + throws InvalidProtocolBufferException { + return parsePartialFrom(input, EMPTY_REGISTRY); + } + + public MessageType parseFrom(InputStream input, + ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + return checkMessageInitialized( + parsePartialFrom(input, extensionRegistry)); + } + + public MessageType parseFrom(InputStream input) + throws InvalidProtocolBufferException { + return parseFrom(input, EMPTY_REGISTRY); + } + + public MessageType parsePartialDelimitedFrom( + InputStream input, + ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + int size; + try { + int firstByte = input.read(); + if (firstByte == -1) { + return null; + } + size = CodedInputStream.readRawVarint32(firstByte, input); + } catch (IOException e) { + throw new InvalidProtocolBufferException(e.getMessage()); + } + InputStream limitedInput = new LimitedInputStream(input, size); + return parsePartialFrom(limitedInput, extensionRegistry); + } + + public MessageType parsePartialDelimitedFrom(InputStream input) + throws InvalidProtocolBufferException { + return parsePartialDelimitedFrom(input, EMPTY_REGISTRY); + } + + public MessageType parseDelimitedFrom( + InputStream input, + ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + return checkMessageInitialized( + parsePartialDelimitedFrom(input, extensionRegistry)); + } + + public MessageType parseDelimitedFrom(InputStream input) + throws InvalidProtocolBufferException { + return parseDelimitedFrom(input, EMPTY_REGISTRY); + } +} diff --git a/java/src/main/java/com/google/protobuf/BoundedByteString.java b/java/src/main/java/com/google/protobuf/BoundedByteString.java new file mode 100644 index 00000000..cd4982c3 --- /dev/null +++ b/java/src/main/java/com/google/protobuf/BoundedByteString.java @@ -0,0 +1,163 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// http://code.google.com/p/protobuf/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import java.util.NoSuchElementException; + +/** + * This class is used to represent the substring of a {@link ByteString} over a + * single byte array. In terms of the public API of {@link ByteString}, you end + * up here by calling {@link ByteString#copyFrom(byte[])} followed by {@link + * ByteString#substring(int, int)}. + * + *

This class contains most of the overhead involved in creating a substring + * from a {@link LiteralByteString}. The overhead involves some range-checking + * and two extra fields. + * + * @author carlanton@google.com (Carl Haverl) + */ +class BoundedByteString extends LiteralByteString { + + private final int bytesOffset; + private final int bytesLength; + + /** + * Creates a {@code BoundedByteString} backed by the sub-range of given array, + * without copying. + * + * @param bytes array to wrap + * @param offset index to first byte to use in bytes + * @param length number of bytes to use from bytes + * @throws IllegalArgumentException if {@code offset < 0}, {@code length < 0}, + * or if {@code offset + length > + * bytes.length}. + */ + BoundedByteString(byte[] bytes, int offset, int length) { + super(bytes); + if (offset < 0) { + throw new IllegalArgumentException("Offset too small: " + offset); + } + if (length < 0) { + throw new IllegalArgumentException("Length too small: " + offset); + } + if ((long) offset + length > bytes.length) { + throw new IllegalArgumentException( + "Offset+Length too large: " + offset + "+" + length); + } + + this.bytesOffset = offset; + this.bytesLength = length; + } + + /** + * Gets the byte at the given index. + * Throws {@link ArrayIndexOutOfBoundsException} + * for backwards-compatibility reasons although it would more properly be + * {@link IndexOutOfBoundsException}. + * + * @param index index of byte + * @return the value + * @throws ArrayIndexOutOfBoundsException {@code index} is < 0 or >= size + */ + @Override + public byte byteAt(int index) { + // We must check the index ourselves as we cannot rely on Java array index + // checking for substrings. + if (index < 0) { + throw new ArrayIndexOutOfBoundsException("Index too small: " + index); + } + if (index >= size()) { + throw new ArrayIndexOutOfBoundsException( + "Index too large: " + index + ", " + size()); + } + + return bytes[bytesOffset + index]; + } + + @Override + public int size() { + return bytesLength; + } + + @Override + protected int getOffsetIntoBytes() { + return bytesOffset; + } + + // ================================================================= + // ByteString -> byte[] + + @Override + protected void copyToInternal(byte[] target, int sourceOffset, + int targetOffset, int numberToCopy) { + System.arraycopy(bytes, getOffsetIntoBytes() + sourceOffset, target, + targetOffset, numberToCopy); + } + + // ================================================================= + // ByteIterator + + @Override + public ByteIterator iterator() { + return new BoundedByteIterator(); + } + + private class BoundedByteIterator implements ByteIterator { + + private int position; + private final int limit; + + private BoundedByteIterator() { + position = getOffsetIntoBytes(); + limit = position + size(); + } + + public boolean hasNext() { + return (position < limit); + } + + public Byte next() { + // Boxing calls Byte.valueOf(byte), which does not instantiate. + return nextByte(); + } + + public byte nextByte() { + if (position >= limit) { + throw new NoSuchElementException(); + } + return bytes[position++]; + } + + public void remove() { + throw new UnsupportedOperationException(); + } + } +} diff --git a/java/src/main/java/com/google/protobuf/ByteString.java b/java/src/main/java/com/google/protobuf/ByteString.java index 91356357..1b18169e 100644 --- a/java/src/main/java/com/google/protobuf/ByteString.java +++ b/java/src/main/java/com/google/protobuf/ByteString.java @@ -30,140 +30,413 @@ package com.google.protobuf; -import java.io.InputStream; -import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; import java.util.List; +import java.util.NoSuchElementException; /** - * Immutable array of bytes. + * Immutable sequence of bytes. Substring is supported by sharing the reference + * to the immutable underlying bytes, as with {@link String}. Concatenation is + * likewise supported without copying (long strings) by building a tree of + * pieces in {@link RopeByteString}. + *

+ * Like {@link String}, the contents of a {@link ByteString} can never be + * observed to change, not even in the presence of a data race or incorrect + * API usage in the client code. * * @author crazybob@google.com Bob Lee * @author kenton@google.com Kenton Varda + * @author carlanton@google.com Carl Haverl + * @author martinrb@google.com Martin Buchholz */ -public final class ByteString { - private final byte[] bytes; +public abstract class ByteString implements Iterable { - private ByteString(final byte[] bytes) { - this.bytes = bytes; - } + /** + * When two strings to be concatenated have a combined length shorter than + * this, we just copy their bytes on {@link #concat(ByteString)}. + * The trade-off is copy size versus the overhead of creating tree nodes + * in {@link RopeByteString}. + */ + static final int CONCATENATE_BY_COPY_SIZE = 128; + + /** + * When copying an InputStream into a ByteString with .readFrom(), + * the chunks in the underlying rope start at 256 bytes, but double + * each iteration up to 8192 bytes. + */ + static final int MIN_READ_FROM_CHUNK_SIZE = 0x100; // 256b + static final int MAX_READ_FROM_CHUNK_SIZE = 0x2000; // 8k /** - * Gets the byte at the given index. + * Empty {@code ByteString}. + */ + public static final ByteString EMPTY = new LiteralByteString(new byte[0]); + + // This constructor is here to prevent subclassing outside of this package, + ByteString() {} + + /** + * Gets the byte at the given index. This method should be used only for + * random access to individual bytes. To access bytes sequentially, use the + * {@link ByteIterator} returned by {@link #iterator()}, and call {@link + * #substring(int, int)} first if necessary. * + * @param index index of byte + * @return the value * @throws ArrayIndexOutOfBoundsException {@code index} is < 0 or >= size */ - public byte byteAt(final int index) { - return bytes[index]; + public abstract byte byteAt(int index); + + /** + * Return a {@link ByteString.ByteIterator} over the bytes in the ByteString. + * To avoid auto-boxing, you may get the iterator manually and call + * {@link ByteIterator#nextByte()}. + * + * @return the iterator + */ + public abstract ByteIterator iterator(); + + /** + * This interface extends {@code Iterator}, so that we can return an + * unboxed {@code byte}. + */ + public interface ByteIterator extends Iterator { + /** + * An alternative to {@link Iterator#next()} that returns an + * unboxed primitive {@code byte}. + * + * @return the next {@code byte} in the iteration + * @throws NoSuchElementException if the iteration has no more elements + */ + byte nextByte(); } /** * Gets the number of bytes. + * + * @return size in bytes */ - public int size() { - return bytes.length; - } + public abstract int size(); /** * Returns {@code true} if the size is {@code 0}, {@code false} otherwise. + * + * @return true if this is zero bytes long */ public boolean isEmpty() { - return bytes.length == 0; + return size() == 0; } // ================================================================= - // byte[] -> ByteString + // ByteString -> substring /** - * Empty ByteString. + * Return the substring from {@code beginIndex}, inclusive, to the end of the + * string. + * + * @param beginIndex start at this index + * @return substring sharing underlying data + * @throws IndexOutOfBoundsException if {@code beginIndex < 0} or + * {@code beginIndex > size()}. */ - public static final ByteString EMPTY = new ByteString(new byte[0]); + public ByteString substring(int beginIndex) { + return substring(beginIndex, size()); + } + + /** + * Return the substring from {@code beginIndex}, inclusive, to {@code + * endIndex}, exclusive. + * + * @param beginIndex start at this index + * @param endIndex the last character is the one before this index + * @return substring sharing underlying data + * @throws IndexOutOfBoundsException if {@code beginIndex < 0}, + * {@code endIndex > size()}, or {@code beginIndex > endIndex}. + */ + public abstract ByteString substring(int beginIndex, int endIndex); + + /** + * Tests if this bytestring starts with the specified prefix. + * Similar to {@link String#startsWith(String)} + * + * @param prefix the prefix. + * @return true if the byte sequence represented by the + * argument is a prefix of the byte sequence represented by + * this string; false otherwise. + */ + public boolean startsWith(ByteString prefix) { + return size() >= prefix.size() && + substring(0, prefix.size()).equals(prefix); + } + + // ================================================================= + // byte[] -> ByteString /** * Copies the given bytes into a {@code ByteString}. + * + * @param bytes source array + * @param offset offset in source array + * @param size number of bytes to copy + * @return new {@code ByteString} */ - public static ByteString copyFrom(final byte[] bytes, final int offset, - final int size) { - final byte[] copy = new byte[size]; + public static ByteString copyFrom(byte[] bytes, int offset, int size) { + byte[] copy = new byte[size]; System.arraycopy(bytes, offset, copy, 0, size); - return new ByteString(copy); + return new LiteralByteString(copy); } /** * Copies the given bytes into a {@code ByteString}. + * + * @param bytes to copy + * @return new {@code ByteString} */ - public static ByteString copyFrom(final byte[] bytes) { + public static ByteString copyFrom(byte[] bytes) { return copyFrom(bytes, 0, bytes.length); } /** - * Copies {@code size} bytes from a {@code java.nio.ByteBuffer} into + * Copies the next {@code size} bytes from a {@code java.nio.ByteBuffer} into * a {@code ByteString}. + * + * @param bytes source buffer + * @param size number of bytes to copy + * @return new {@code ByteString} */ - public static ByteString copyFrom(final ByteBuffer bytes, final int size) { - final byte[] copy = new byte[size]; + public static ByteString copyFrom(ByteBuffer bytes, int size) { + byte[] copy = new byte[size]; bytes.get(copy); - return new ByteString(copy); + return new LiteralByteString(copy); } /** * Copies the remaining bytes from a {@code java.nio.ByteBuffer} into * a {@code ByteString}. + * + * @param bytes sourceBuffer + * @return new {@code ByteString} */ - public static ByteString copyFrom(final ByteBuffer bytes) { + public static ByteString copyFrom(ByteBuffer bytes) { return copyFrom(bytes, bytes.remaining()); } /** * Encodes {@code text} into a sequence of bytes using the named charset * and returns the result as a {@code ByteString}. + * + * @param text source string + * @param charsetName encoding to use + * @return new {@code ByteString} + * @throws UnsupportedEncodingException if the encoding isn't found */ - public static ByteString copyFrom(final String text, final String charsetName) + public static ByteString copyFrom(String text, String charsetName) throws UnsupportedEncodingException { - return new ByteString(text.getBytes(charsetName)); + return new LiteralByteString(text.getBytes(charsetName)); } /** * Encodes {@code text} into a sequence of UTF-8 bytes and returns the * result as a {@code ByteString}. + * + * @param text source string + * @return new {@code ByteString} */ - public static ByteString copyFromUtf8(final String text) { + public static ByteString copyFromUtf8(String text) { try { - return new ByteString(text.getBytes("UTF-8")); + return new LiteralByteString(text.getBytes("UTF-8")); } catch (UnsupportedEncodingException e) { throw new RuntimeException("UTF-8 not supported?", e); } } + // ================================================================= + // InputStream -> ByteString + /** - * Concatenates all byte strings in the list and returns the result. + * Completely reads the given stream's bytes into a + * {@code ByteString}, blocking if necessary until all bytes are + * read through to the end of the stream. + * + * Performance notes: The returned {@code ByteString} is an + * immutable tree of byte arrays ("chunks") of the stream data. The + * first chunk is small, with subsequent chunks each being double + * the size, up to 8K. If the caller knows the precise length of + * the stream and wishes to avoid all unnecessary copies and + * allocations, consider using the two-argument version of this + * method, below. + * + * @param streamToDrain The source stream, which is read completely + * but not closed. + * @return A new {@code ByteString} which is made up of chunks of + * various sizes, depending on the behavior of the underlying + * stream. + * @throws IOException IOException is thrown if there is a problem + * reading the underlying stream. + */ + public static ByteString readFrom(InputStream streamToDrain) + throws IOException { + return readFrom( + streamToDrain, MIN_READ_FROM_CHUNK_SIZE, MAX_READ_FROM_CHUNK_SIZE); + } + + /** + * Completely reads the given stream's bytes into a + * {@code ByteString}, blocking if necessary until all bytes are + * read through to the end of the stream. + * + * Performance notes: The returned {@code ByteString} is an + * immutable tree of byte arrays ("chunks") of the stream data. The + * chunkSize parameter sets the size of these byte arrays. In + * particular, if the chunkSize is precisely the same as the length + * of the stream, unnecessary allocations and copies will be + * avoided. Otherwise, the chunks will be of the given size, except + * for the last chunk, which will be resized (via a reallocation and + * copy) to contain the remainder of the stream. + * + * @param streamToDrain The source stream, which is read completely + * but not closed. + * @param chunkSize The size of the chunks in which to read the + * stream. + * @return A new {@code ByteString} which is made up of chunks of + * the given size. + * @throws IOException IOException is thrown if there is a problem + * reading the underlying stream. + */ + public static ByteString readFrom(InputStream streamToDrain, int chunkSize) + throws IOException { + return readFrom(streamToDrain, chunkSize, chunkSize); + } + + // Helper method that takes the chunk size range as a parameter. + public static ByteString readFrom(InputStream streamToDrain, int minChunkSize, + int maxChunkSize) throws IOException { + Collection results = new ArrayList(); + + // copy the inbound bytes into a list of chunks; the chunk size + // grows exponentially to support both short and long streams. + int chunkSize = minChunkSize; + while (true) { + ByteString chunk = readChunk(streamToDrain, chunkSize); + if (chunk == null) { + break; + } + results.add(chunk); + chunkSize = Math.min(chunkSize * 2, maxChunkSize); + } + + return ByteString.copyFrom(results); + } + + /** + * Blocks until a chunk of the given size can be made from the + * stream, or EOF is reached. Calls read() repeatedly in case the + * given stream implementation doesn't completely fill the given + * buffer in one read() call. + * + * @return A chunk of the desired size, or else a chunk as large as + * was available when end of stream was reached. Returns null if the + * given stream had no more data in it. + */ + private static ByteString readChunk(InputStream in, final int chunkSize) + throws IOException { + final byte[] buf = new byte[chunkSize]; + int bytesRead = 0; + while (bytesRead < chunkSize) { + final int count = in.read(buf, bytesRead, chunkSize - bytesRead); + if (count == -1) { + break; + } + bytesRead += count; + } + + if (bytesRead == 0) { + return null; + } else { + return ByteString.copyFrom(buf, 0, bytesRead); + } + } + + // ================================================================= + // Multiple ByteStrings -> One ByteString + + /** + * Concatenate the given {@code ByteString} to this one. Short concatenations, + * of total size smaller than {@link ByteString#CONCATENATE_BY_COPY_SIZE}, are + * produced by copying the underlying bytes (as per Rope.java, + * BAP95 . In general, the concatenate involves no copying. + * + * @param other string to concatenate + * @return a new {@code ByteString} instance + */ + public ByteString concat(ByteString other) { + int thisSize = size(); + int otherSize = other.size(); + if ((long) thisSize + otherSize >= Integer.MAX_VALUE) { + throw new IllegalArgumentException("ByteString would be too long: " + + thisSize + "+" + otherSize); + } + + return RopeByteString.concatenate(this, other); + } + + /** + * Concatenates all byte strings in the iterable and returns the result. + * This is designed to run in O(list size), not O(total bytes). * *

The returned {@code ByteString} is not necessarily a unique object. * If the list is empty, the returned object is the singleton empty * {@code ByteString}. If the list has only one element, that * {@code ByteString} will be returned without copying. + * + * @param byteStrings strings to be concatenated + * @return new {@code ByteString} */ - public static ByteString copyFrom(List list) { - if (list.size() == 0) { - return EMPTY; - } else if (list.size() == 1) { - return list.get(0); + public static ByteString copyFrom(Iterable byteStrings) { + Collection collection; + if (!(byteStrings instanceof Collection)) { + collection = new ArrayList(); + for (ByteString byteString : byteStrings) { + collection.add(byteString); + } + } else { + collection = (Collection) byteStrings; } - - int size = 0; - for (ByteString str : list) { - size += str.size(); + ByteString result; + if (collection.isEmpty()) { + result = EMPTY; + } else { + result = balancedConcat(collection.iterator(), collection.size()); } - byte[] bytes = new byte[size]; - int pos = 0; - for (ByteString str : list) { - System.arraycopy(str.bytes, 0, bytes, pos, str.size()); - pos += str.size(); + return result; + } + + // Internal function used by copyFrom(Iterable). + // Create a balanced concatenation of the next "length" elements from the + // iterable. + private static ByteString balancedConcat(Iterator iterator, + int length) { + assert length >= 1; + ByteString result; + if (length == 1) { + result = iterator.next(); + } else { + int halfLength = length >>> 1; + ByteString left = balancedConcat(iterator, halfLength); + ByteString right = balancedConcat(iterator, length - halfLength); + result = left.concat(right); } - return new ByteString(bytes); + return result; } // ================================================================= @@ -174,206 +447,446 @@ public final class ByteString { * * @param target buffer to copy into * @param offset in the target buffer + * @throws IndexOutOfBoundsException if the offset is negative or too large */ - public void copyTo(final byte[] target, final int offset) { - System.arraycopy(bytes, 0, target, offset, bytes.length); + public void copyTo(byte[] target, int offset) { + copyTo(target, 0, offset, size()); } /** * Copies bytes into a buffer. * - * @param target buffer to copy into + * @param target buffer to copy into * @param sourceOffset offset within these bytes * @param targetOffset offset within the target buffer - * @param size number of bytes to copy + * @param numberToCopy number of bytes to copy + * @throws IndexOutOfBoundsException if an offset or size is negative or too + * large */ - public void copyTo(final byte[] target, final int sourceOffset, - final int targetOffset, - final int size) { - System.arraycopy(bytes, sourceOffset, target, targetOffset, size); + public void copyTo(byte[] target, int sourceOffset, int targetOffset, + int numberToCopy) { + if (sourceOffset < 0) { + throw new IndexOutOfBoundsException("Source offset < 0: " + sourceOffset); + } + if (targetOffset < 0) { + throw new IndexOutOfBoundsException("Target offset < 0: " + targetOffset); + } + if (numberToCopy < 0) { + throw new IndexOutOfBoundsException("Length < 0: " + numberToCopy); + } + if (sourceOffset + numberToCopy > size()) { + throw new IndexOutOfBoundsException( + "Source end offset < 0: " + (sourceOffset + numberToCopy)); + } + if (targetOffset + numberToCopy > target.length) { + throw new IndexOutOfBoundsException( + "Target end offset < 0: " + (targetOffset + numberToCopy)); + } + if (numberToCopy > 0) { + copyToInternal(target, sourceOffset, targetOffset, numberToCopy); + } } + /** + * Internal (package private) implementation of + * @link{#copyTo(byte[],int,int,int}. + * It assumes that all error checking has already been performed and that + * @code{numberToCopy > 0}. + */ + protected abstract void copyToInternal(byte[] target, int sourceOffset, + int targetOffset, int numberToCopy); + /** * Copies bytes into a ByteBuffer. * * @param target ByteBuffer to copy into. - * @throws ReadOnlyBufferException if the {@code target} is read-only - * @throws BufferOverflowException if the {@code target}'s remaining() - * space is not large enough to hold the data. + * @throws java.nio.ReadOnlyBufferException if the {@code target} is read-only + * @throws java.nio.BufferOverflowException if the {@code target}'s + * remaining() space is not large enough to hold the data. */ - public void copyTo(ByteBuffer target) { - target.put(bytes, 0, bytes.length); - } + public abstract void copyTo(ByteBuffer target); /** * Copies bytes to a {@code byte[]}. + * + * @return copied bytes */ public byte[] toByteArray() { - final int size = bytes.length; - final byte[] copy = new byte[size]; - System.arraycopy(bytes, 0, copy, 0, size); - return copy; + int size = size(); + byte[] result = new byte[size]; + copyToInternal(result, 0, 0, size); + return result; } /** - * Constructs a new read-only {@code java.nio.ByteBuffer} with the - * same backing byte array. + * Writes the complete contents of this byte string to + * the specified output stream argument. + * + * @param out the output stream to which to write the data. + * @throws IOException if an I/O error occurs. */ - public ByteBuffer asReadOnlyByteBuffer() { - final ByteBuffer byteBuffer = ByteBuffer.wrap(bytes); - return byteBuffer.asReadOnlyBuffer(); - } + public abstract void writeTo(OutputStream out) throws IOException; + + /** + * Constructs a read-only {@code java.nio.ByteBuffer} whose content + * is equal to the contents of this byte string. + * The result uses the same backing array as the byte string, if possible. + * + * @return wrapped bytes + */ + public abstract ByteBuffer asReadOnlyByteBuffer(); + + /** + * Constructs a list of read-only {@code java.nio.ByteBuffer} objects + * such that the concatenation of their contents is equal to the contents + * of this byte string. The result uses the same backing arrays as the + * byte string. + *

+ * By returning a list, implementations of this method may be able to avoid + * copying even when there are multiple backing arrays. + * + * @return a list of wrapped bytes + */ + public abstract List asReadOnlyByteBufferList(); /** * Constructs a new {@code String} by decoding the bytes using the * specified charset. + * + * @param charsetName encode using this charset + * @return new string + * @throws UnsupportedEncodingException if charset isn't recognized */ - public String toString(final String charsetName) - throws UnsupportedEncodingException { - return new String(bytes, charsetName); - } + public abstract String toString(String charsetName) + throws UnsupportedEncodingException; + + // ================================================================= + // UTF-8 decoding /** * Constructs a new {@code String} by decoding the bytes as UTF-8. + * + * @return new string using UTF-8 encoding */ public String toStringUtf8() { try { - return new String(bytes, "UTF-8"); + return toString("UTF-8"); } catch (UnsupportedEncodingException e) { throw new RuntimeException("UTF-8 not supported?", e); } } + /** + * Tells whether this {@code ByteString} represents a well-formed UTF-8 + * byte sequence, such that the original bytes can be converted to a + * String object and then round tripped back to bytes without loss. + * + *

More precisely, returns {@code true} whenever: 

 {@code
+   * Arrays.equals(byteString.toByteArray(),
+   *     new String(byteString.toByteArray(), "UTF-8").getBytes("UTF-8"))
+   * }
+ * + *

This method returns {@code false} for "overlong" byte sequences, + * as well as for 3-byte sequences that would map to a surrogate + * character, in accordance with the restricted definition of UTF-8 + * introduced in Unicode 3.1. Note that the UTF-8 decoder included in + * Oracle's JDK has been modified to also reject "overlong" byte + * sequences, but (as of 2011) still accepts 3-byte surrogate + * character byte sequences. + * + *

See the Unicode Standard,
+ * Table 3-6. UTF-8 Bit Distribution,
+ * Table 3-7. Well Formed UTF-8 Byte Sequences. + * + * @return whether the bytes in this {@code ByteString} are a + * well-formed UTF-8 byte sequence + */ + public abstract boolean isValidUtf8(); + + /** + * Tells whether the given byte sequence is a well-formed, malformed, or + * incomplete UTF-8 byte sequence. This method accepts and returns a partial + * state result, allowing the bytes for a complete UTF-8 byte sequence to be + * composed from multiple {@code ByteString} segments. + * + * @param state either {@code 0} (if this is the initial decoding operation) + * or the value returned from a call to a partial decoding method for the + * previous bytes + * @param offset offset of the first byte to check + * @param length number of bytes to check + * + * @return {@code -1} if the partial byte sequence is definitely malformed, + * {@code 0} if it is well-formed (no additional input needed), or, if the + * byte sequence is "incomplete", i.e. apparently terminated in the middle of + * a character, an opaque integer "state" value containing enough information + * to decode the character when passed to a subsequent invocation of a + * partial decoding method. + */ + protected abstract int partialIsValidUtf8(int state, int offset, int length); + // ================================================================= // equals() and hashCode() @Override - public boolean equals(final Object o) { - if (o == this) { - return true; - } - - if (!(o instanceof ByteString)) { - return false; - } - - final ByteString other = (ByteString) o; - final int size = bytes.length; - if (size != other.bytes.length) { - return false; - } - - final byte[] thisBytes = bytes; - final byte[] otherBytes = other.bytes; - for (int i = 0; i < size; i++) { - if (thisBytes[i] != otherBytes[i]) { - return false; - } - } - - return true; - } - - private volatile int hash = 0; + public abstract boolean equals(Object o); + /** + * Return a non-zero hashCode depending only on the sequence of bytes + * in this ByteString. + * + * @return hashCode value for this object + */ @Override - public int hashCode() { - int h = hash; - - if (h == 0) { - final byte[] thisBytes = bytes; - final int size = bytes.length; - - h = size; - for (int i = 0; i < size; i++) { - h = h * 31 + thisBytes[i]; - } - if (h == 0) { - h = 1; - } - - hash = h; - } - - return h; - } + public abstract int hashCode(); // ================================================================= // Input stream /** * Creates an {@code InputStream} which can be used to read the bytes. + *

+ * The {@link InputStream} returned by this method is guaranteed to be + * completely non-blocking. The method {@link InputStream#available()} + * returns the number of bytes remaining in the stream. The methods + * {@link InputStream#read(byte[]), {@link InputStream#read(byte[],int,int)} + * and {@link InputStream#skip(long)} will read/skip as many bytes as are + * available. + *

+ * The methods in the returned {@link InputStream} might not be + * thread safe. + * + * @return an input stream that returns the bytes of this byte string. */ - public InputStream newInput() { - return new ByteArrayInputStream(bytes); - } + public abstract InputStream newInput(); /** * Creates a {@link CodedInputStream} which can be used to read the bytes. - * Using this is more efficient than creating a {@link CodedInputStream} - * wrapping the result of {@link #newInput()}. + * Using this is often more efficient than creating a {@link CodedInputStream} + * that wraps the result of {@link #newInput()}. + * + * @return stream based on wrapped data */ - public CodedInputStream newCodedInput() { - // We trust CodedInputStream not to modify the bytes, or to give anyone - // else access to them. - return CodedInputStream.newInstance(bytes); - } + public abstract CodedInputStream newCodedInput(); // ================================================================= // Output stream /** - * Creates a new {@link Output} with the given initial capacity. + * Creates a new {@link Output} with the given initial capacity. Call {@link + * Output#toByteString()} to create the {@code ByteString} instance. + *

+ * A {@link ByteString.Output} offers the same functionality as a + * {@link ByteArrayOutputStream}, except that it returns a {@link ByteString} + * rather than a {@code byte} array. + * + * @param initialCapacity estimate of number of bytes to be written + * @return {@code OutputStream} for building a {@code ByteString} */ - public static Output newOutput(final int initialCapacity) { - return new Output(new ByteArrayOutputStream(initialCapacity)); + public static Output newOutput(int initialCapacity) { + return new Output(initialCapacity); } /** - * Creates a new {@link Output}. + * Creates a new {@link Output}. Call {@link Output#toByteString()} to create + * the {@code ByteString} instance. + *

+ * A {@link ByteString.Output} offers the same functionality as a + * {@link ByteArrayOutputStream}, except that it returns a {@link ByteString} + * rather than a {@code byte array}. + * + * @return {@code OutputStream} for building a {@code ByteString} */ public static Output newOutput() { - return newOutput(32); + return new Output(CONCATENATE_BY_COPY_SIZE); } /** * Outputs to a {@code ByteString} instance. Call {@link #toByteString()} to * create the {@code ByteString} instance. */ - public static final class Output extends FilterOutputStream { - private final ByteArrayOutputStream bout; + public static final class Output extends OutputStream { + // Implementation note. + // The public methods of this class must be synchronized. ByteStrings + // are guaranteed to be immutable. Without some sort of locking, it could + // be possible for one thread to call toByteSring(), while another thread + // is still modifying the underlying byte array. + + private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + // argument passed by user, indicating initial capacity. + private final int initialCapacity; + // ByteStrings to be concatenated to create the result + private final ArrayList flushedBuffers; + // Total number of bytes in the ByteStrings of flushedBuffers + private int flushedBuffersTotalBytes; + // Current buffer to which we are writing + private byte[] buffer; + // Location in buffer[] to which we write the next byte. + private int bufferPos; /** - * Constructs a new output with the given initial capacity. + * Creates a new ByteString output stream with the specified + * initial capacity. + * + * @param initialCapacity the initial capacity of the output stream. */ - private Output(final ByteArrayOutputStream bout) { - super(bout); - this.bout = bout; + Output(int initialCapacity) { + if (initialCapacity < 0) { + throw new IllegalArgumentException("Buffer size < 0"); + } + this.initialCapacity = initialCapacity; + this.flushedBuffers = new ArrayList(); + this.buffer = new byte[initialCapacity]; + } + + @Override + public synchronized void write(int b) { + if (bufferPos == buffer.length) { + flushFullBuffer(1); + } + buffer[bufferPos++] = (byte)b; + } + + @Override + public synchronized void write(byte[] b, int offset, int length) { + if (length <= buffer.length - bufferPos) { + // The bytes can fit into the current buffer. + System.arraycopy(b, offset, buffer, bufferPos, length); + bufferPos += length; + } else { + // Use up the current buffer + int copySize = buffer.length - bufferPos; + System.arraycopy(b, offset, buffer, bufferPos, copySize); + offset += copySize; + length -= copySize; + // Flush the buffer, and get a new buffer at least big enough to cover + // what we still need to output + flushFullBuffer(length); + System.arraycopy(b, offset, buffer, 0 /* count */, length); + bufferPos = length; + } + } + + /** + * Creates a byte string. Its size is the current size of this output + * stream and its output has been copied to it. + * + * @return the current contents of this output stream, as a byte string. + */ + public synchronized ByteString toByteString() { + flushLastBuffer(); + return ByteString.copyFrom(flushedBuffers); + } + + /** + * Writes the complete contents of this byte array output stream to + * the specified output stream argument. + * + * @param out the output stream to which to write the data. + * @throws IOException if an I/O error occurs. + */ + public void writeTo(OutputStream out) throws IOException { + ByteString[] cachedFlushBuffers; + byte[] cachedBuffer; + int cachedBufferPos; + synchronized (this) { + // Copy the information we need into local variables so as to hold + // the lock for as short a time as possible. + cachedFlushBuffers = + flushedBuffers.toArray(new ByteString[flushedBuffers.size()]); + cachedBuffer = buffer; + cachedBufferPos = bufferPos; + } + for (ByteString byteString : cachedFlushBuffers) { + byteString.writeTo(out); + } + + out.write(Arrays.copyOf(cachedBuffer, cachedBufferPos)); + } + + /** + * Returns the current size of the output stream. + * + * @return the current size of the output stream + */ + public synchronized int size() { + return flushedBuffersTotalBytes + bufferPos; + } + + /** + * Resets this stream, so that all currently accumulated output in the + * output stream is discarded. The output stream can be used again, + * reusing the already allocated buffer space. + */ + public synchronized void reset() { + flushedBuffers.clear(); + flushedBuffersTotalBytes = 0; + bufferPos = 0; + } + + @Override + public String toString() { + return String.format("", + Integer.toHexString(System.identityHashCode(this)), size()); } /** - * Creates a {@code ByteString} instance from this {@code Output}. + * Internal function used by writers. The current buffer is full, and the + * writer needs a new buffer whose size is at least the specified minimum + * size. */ - public ByteString toByteString() { - final byte[] byteArray = bout.toByteArray(); - return new ByteString(byteArray); + private void flushFullBuffer(int minSize) { + flushedBuffers.add(new LiteralByteString(buffer)); + flushedBuffersTotalBytes += buffer.length; + // We want to increase our total capacity by 50%, but as a minimum, + // the new buffer should also at least be >= minSize and + // >= initial Capacity. + int newSize = Math.max(initialCapacity, + Math.max(minSize, flushedBuffersTotalBytes >>> 1)); + buffer = new byte[newSize]; + bufferPos = 0; + } + + /** + * Internal function used by {@link #toByteString()}. The current buffer may + * or may not be full, but it needs to be flushed. + */ + private void flushLastBuffer() { + if (bufferPos < buffer.length) { + if (bufferPos > 0) { + byte[] bufferCopy = Arrays.copyOf(buffer, bufferPos); + flushedBuffers.add(new LiteralByteString(bufferCopy)); + } + // We reuse this buffer for further writes. + } else { + // Buffer is completely full. Huzzah. + flushedBuffers.add(new LiteralByteString(buffer)); + // 99% of the time, we're not going to use this OutputStream again. + // We set buffer to an empty byte stream so that we're handling this + // case without wasting space. In the rare case that more writes + // *do* occur, this empty buffer will be flushed and an appropriately + // sized new buffer will be created. + buffer = EMPTY_BYTE_ARRAY; + } + flushedBuffersTotalBytes += bufferPos; + bufferPos = 0; } } /** - * Constructs a new ByteString builder, which allows you to efficiently - * construct a {@code ByteString} by writing to a {@link CodedOutputStream}. - * Using this is much more efficient than calling {@code newOutput()} and - * wrapping that in a {@code CodedOutputStream}. + * Constructs a new {@code ByteString} builder, which allows you to + * efficiently construct a {@code ByteString} by writing to a {@link + * CodedOutputStream}. Using this is much more efficient than calling {@code + * newOutput()} and wrapping that in a {@code CodedOutputStream}. * *

This is package-private because it's a somewhat confusing interface. * Users can call {@link Message#toByteString()} instead of calling this * directly. * - * @param size The target byte size of the {@code ByteString}. You must - * write exactly this many bytes before building the result. + * @param size The target byte size of the {@code ByteString}. You must write + * exactly this many bytes before building the result. + * @return the builder */ - static CodedBuilder newCodedBuilder(final int size) { + static CodedBuilder newCodedBuilder(int size) { return new CodedBuilder(size); } @@ -382,7 +895,7 @@ public final class ByteString { private final CodedOutputStream output; private final byte[] buffer; - private CodedBuilder(final int size) { + private CodedBuilder(int size) { buffer = new byte[size]; output = CodedOutputStream.newInstance(buffer); } @@ -393,11 +906,57 @@ public final class ByteString { // We can be confident that the CodedOutputStream will not modify the // underlying bytes anymore because it already wrote all of them. So, // no need to make a copy. - return new ByteString(buffer); + return new LiteralByteString(buffer); } public CodedOutputStream getCodedOutput() { return output; } } + + // ================================================================= + // Methods {@link RopeByteString} needs on instances, which aren't part of the + // public API. + + /** + * Return the depth of the tree representing this {@code ByteString}, if any, + * whose root is this node. If this is a leaf node, return 0. + * + * @return tree depth or zero + */ + protected abstract int getTreeDepth(); + + /** + * Return {@code true} if this ByteString is literal (a leaf node) or a + * flat-enough tree in the sense of {@link RopeByteString}. + * + * @return true if the tree is flat enough + */ + protected abstract boolean isBalanced(); + + /** + * Return the cached hash code if available. + * + * @return value of cached hash code or 0 if not computed yet + */ + protected abstract int peekCachedHashCode(); + + /** + * Compute the hash across the value bytes starting with the given hash, and + * return the result. This is used to compute the hash across strings + * represented as a set of pieces by allowing the hash computation to be + * continued from piece to piece. + * + * @param h starting hash value + * @param offset offset into this value to start looking at data values + * @param length number of data values to include in the hash computation + * @return ending hash value + */ + protected abstract int partialHash(int h, int offset, int length); + + @Override + public String toString() { + return String.format("", + Integer.toHexString(System.identityHashCode(this)), size()); + } } diff --git a/java/src/main/java/com/google/protobuf/CodedInputStream.java b/java/src/main/java/com/google/protobuf/CodedInputStream.java index b3e08555..33417a7f 100644 --- a/java/src/main/java/com/google/protobuf/CodedInputStream.java +++ b/java/src/main/java/com/google/protobuf/CodedInputStream.java @@ -243,6 +243,23 @@ public final class CodedInputStream { --recursionDepth; } + /** Read a {@code group} field value from the stream. */ + public T readGroup( + final int fieldNumber, + final Parser parser, + final ExtensionRegistryLite extensionRegistry) + throws IOException { + if (recursionDepth >= recursionLimit) { + throw InvalidProtocolBufferException.recursionLimitExceeded(); + } + ++recursionDepth; + T result = parser.parsePartialFrom(this, extensionRegistry); + checkLastTagWas( + WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); + --recursionDepth; + return result; + } + /** * Reads a {@code group} field value from the stream and merges it into the * given {@link UnknownFieldSet}. @@ -278,6 +295,24 @@ public final class CodedInputStream { popLimit(oldLimit); } + /** Read an embedded message field value from the stream. */ + public T readMessage( + final Parser parser, + final ExtensionRegistryLite extensionRegistry) + throws IOException { + int length = readRawVarint32(); + if (recursionDepth >= recursionLimit) { + throw InvalidProtocolBufferException.recursionLimitExceeded(); + } + final int oldLimit = pushLimit(length); + ++recursionDepth; + T result = parser.parsePartialFrom(this, extensionRegistry); + checkLastTagWas(0); + --recursionDepth; + popLimit(oldLimit); + return result; + } + /** Read a {@code bytes} field value from the stream. */ public ByteString readBytes() throws IOException { final int size = readRawVarint32(); @@ -601,7 +636,7 @@ public final class CodedInputStream { * refreshing its buffer. If you need to prevent reading past a certain * point in the underlying {@code InputStream} (e.g. because you expect it to * contain more data after the end of the message which you need to handle - * differently) then you must place a wrapper around you {@code InputStream} + * differently) then you must place a wrapper around your {@code InputStream} * which limits the amount of data that can be read from it. * * @return the old limit. @@ -676,7 +711,7 @@ public final class CodedInputStream { /** * Called with {@code this.buffer} is empty to read more bytes from the - * input. If {@code mustSucceed} is true, refillBuffer() gurantees that + * input. If {@code mustSucceed} is true, refillBuffer() guarantees that * either there will be at least one byte in the buffer when it returns * or it will throw an exception. If {@code mustSucceed} is false, * refillBuffer() returns false if no more bytes were available. @@ -879,7 +914,7 @@ public final class CodedInputStream { refillBuffer(true); } - bufferPos = size - pos; + bufferPos = size - pos; } } } diff --git a/java/src/main/java/com/google/protobuf/CodedOutputStream.java b/java/src/main/java/com/google/protobuf/CodedOutputStream.java index ac5f2d30..ca24638d 100644 --- a/java/src/main/java/com/google/protobuf/CodedOutputStream.java +++ b/java/src/main/java/com/google/protobuf/CodedOutputStream.java @@ -30,10 +30,10 @@ package com.google.protobuf; -import java.io.OutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.io.InputStream; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; /** * Encodes and writes protocol message fields. @@ -539,6 +539,15 @@ public final class CodedOutputStream { return computeTagSize(fieldNumber) + computeBytesSizeNoTag(value); } + /** + * Compute the number of bytes that would be needed to encode an + * embedded message in lazy field, including tag. + */ + public static int computeLazyFieldSize(final int fieldNumber, + final LazyField value) { + return computeTagSize(fieldNumber) + computeLazyFieldSizeNoTag(value); + } + /** * Compute the number of bytes that would be needed to encode a * {@code uint32} field, including tag. @@ -614,6 +623,18 @@ public final class CodedOutputStream { computeBytesSize(WireFormat.MESSAGE_SET_MESSAGE, value); } + /** + * Compute the number of bytes that would be needed to encode an + * lazily parsed MessageSet extension field to the stream. For + * historical reasons, the wire format differs from normal fields. + */ + public static int computeLazyFieldMessageSetExtensionSize( + final int fieldNumber, final LazyField value) { + return computeTagSize(WireFormat.MESSAGE_SET_ITEM) * 2 + + computeUInt32Size(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber) + + computeLazyFieldSize(WireFormat.MESSAGE_SET_MESSAGE, value); + } + // ----------------------------------------------------------------- /** @@ -729,6 +750,15 @@ public final class CodedOutputStream { return computeRawVarint32Size(size) + size; } + /** + * Compute the number of bytes that would be needed to encode an embedded + * message stored in lazy field. + */ + public static int computeLazyFieldSizeNoTag(final LazyField value) { + final int size = value.getSerializedSize(); + return computeRawVarint32Size(size) + size; + } + /** * Compute the number of bytes that would be needed to encode a * {@code bytes} field. diff --git a/java/src/main/java/com/google/protobuf/Descriptors.java b/java/src/main/java/com/google/protobuf/Descriptors.java index 2ee84594..a4913053 100644 --- a/java/src/main/java/com/google/protobuf/Descriptors.java +++ b/java/src/main/java/com/google/protobuf/Descriptors.java @@ -35,8 +35,10 @@ import com.google.protobuf.DescriptorProtos.*; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.io.UnsupportedEncodingException; /** @@ -106,6 +108,11 @@ public final class Descriptors { return Collections.unmodifiableList(Arrays.asList(dependencies)); } + /** Get a list of this file's public dependencies (public imports). */ + public List getPublicDependencies() { + return Collections.unmodifiableList(Arrays.asList(publicDependencies)); + } + /** * Find a message type in the file by name. Does not find nested types. * @@ -216,7 +223,7 @@ public final class Descriptors { public static FileDescriptor buildFrom(final FileDescriptorProto proto, final FileDescriptor[] dependencies) throws DescriptorValidationException { - // Building decsriptors involves two steps: translating and linking. + // Building descriptors involves two steps: translating and linking. // In the translation step (implemented by FileDescriptor's // constructor), we build an object tree mirroring the // FileDescriptorProto's tree and put all of the descriptors into the @@ -317,12 +324,12 @@ public final class Descriptors { * {@link FileDescriptor#internalBuildGeneratedFileFrom}, the caller * provides a callback implementing this interface. The callback is called * after the FileDescriptor has been constructed, in order to assign all - * the global variales defined in the generated code which point at parts + * the global variables defined in the generated code which point at parts * of the FileDescriptor. The callback returns an ExtensionRegistry which * contains any extensions which might be used in the descriptor -- that * is, extensions of the various "Options" messages defined in * descriptor.proto. The callback may also return null to indicate that - * no extensions are used in the decsriptor. + * no extensions are used in the descriptor. */ public interface InternalDescriptorAssigner { ExtensionRegistry assignDescriptors(FileDescriptor root); @@ -334,6 +341,7 @@ public final class Descriptors { private final ServiceDescriptor[] services; private final FieldDescriptor[] extensions; private final FileDescriptor[] dependencies; + private final FileDescriptor[] publicDependencies; private final DescriptorPool pool; private FileDescriptor(final FileDescriptorProto proto, @@ -343,6 +351,17 @@ public final class Descriptors { this.pool = pool; this.proto = proto; this.dependencies = dependencies.clone(); + this.publicDependencies = + new FileDescriptor[proto.getPublicDependencyCount()]; + for (int i = 0; i < proto.getPublicDependencyCount(); i++) { + int index = proto.getPublicDependency(i); + if (index < 0 || index >= this.dependencies.length) { + throw new DescriptorValidationException(this, + "Invalid public dependency index."); + } + this.publicDependencies[i] = + this.dependencies[proto.getPublicDependency(i)]; + } pool.addPackage(getPackage(), this); @@ -390,7 +409,7 @@ public final class Descriptors { * in the original. This method is needed for bootstrapping when a file * defines custom options. The options may be defined in the file itself, * so we can't actually parse them until we've constructed the descriptors, - * but to construct the decsriptors we have to have parsed the descriptor + * but to construct the descriptors we have to have parsed the descriptor * protos. So, we have to parse the descriptor protos a second time after * constructing the descriptors. */ @@ -641,7 +660,7 @@ public final class Descriptors { FieldSet.FieldDescriptorLite { /** * Get the index of this descriptor within its parent. - * @see Descriptor#getIndex() + * @see Descriptors.Descriptor#getIndex() */ public int getIndex() { return index; } @@ -656,7 +675,7 @@ public final class Descriptors { /** * Get the field's fully-qualified name. - * @see Descriptor#getFullName() + * @see Descriptors.Descriptor#getFullName() */ public String getFullName() { return fullName; } @@ -943,7 +962,8 @@ public final class Descriptors { private void crossLink() throws DescriptorValidationException { if (proto.hasExtendee()) { final GenericDescriptor extendee = - file.pool.lookupSymbol(proto.getExtendee(), this); + file.pool.lookupSymbol(proto.getExtendee(), this, + DescriptorPool.SearchFilter.TYPES_ONLY); if (!(extendee instanceof Descriptor)) { throw new DescriptorValidationException(this, '\"' + proto.getExtendee() + "\" is not a message type."); @@ -960,7 +980,8 @@ public final class Descriptors { if (proto.hasTypeName()) { final GenericDescriptor typeDescriptor = - file.pool.lookupSymbol(proto.getTypeName(), this); + file.pool.lookupSymbol(proto.getTypeName(), this, + DescriptorPool.SearchFilter.TYPES_ONLY); if (!proto.hasType()) { // Choose field type based on symbol. @@ -1149,7 +1170,7 @@ public final class Descriptors { implements GenericDescriptor, Internal.EnumLiteMap { /** * Get the index of this descriptor within its parent. - * @see Descriptor#getIndex() + * @see Descriptors.Descriptor#getIndex() */ public int getIndex() { return index; } @@ -1161,7 +1182,7 @@ public final class Descriptors { /** * Get the type's fully-qualified name. - * @see Descriptor#getFullName() + * @see Descriptors.Descriptor#getFullName() */ public String getFullName() { return fullName; } @@ -1182,7 +1203,7 @@ public final class Descriptors { /** * Find an enum value by name. * @param name The unqualified name of the value (e.g. "FOO"). - * @return the value's decsriptor, or {@code null} if not found. + * @return the value's descriptor, or {@code null} if not found. */ public EnumValueDescriptor findValueByName(final String name) { final GenericDescriptor result = @@ -1198,7 +1219,7 @@ public final class Descriptors { * Find an enum value by number. If multiple enum values have the same * number, this returns the first defined value with that number. * @param number The value's number. - * @return the value's decsriptor, or {@code null} if not found. + * @return the value's descriptor, or {@code null} if not found. */ public EnumValueDescriptor findValueByNumber(final int number) { return file.pool.enumValuesByNumber.get( @@ -1261,7 +1282,7 @@ public final class Descriptors { implements GenericDescriptor, Internal.EnumLite { /** * Get the index of this descriptor within its parent. - * @see Descriptor#getIndex() + * @see Descriptors.Descriptor#getIndex() */ public int getIndex() { return index; } @@ -1276,7 +1297,7 @@ public final class Descriptors { /** * Get the value's fully-qualified name. - * @see Descriptor#getFullName() + * @see Descriptors.Descriptor#getFullName() */ public String getFullName() { return fullName; } @@ -1337,7 +1358,7 @@ public final class Descriptors { /** * Get the type's fully-qualified name. - * @see Descriptor#getFullName() + * @see Descriptors.Descriptor#getFullName() */ public String getFullName() { return fullName; } @@ -1355,7 +1376,7 @@ public final class Descriptors { /** * Find a method by name. * @param name The unqualified name of the method (e.g. "Foo"). - * @return the method's decsriptor, or {@code null} if not found. + * @return the method's descriptor, or {@code null} if not found. */ public MethodDescriptor findMethodByName(final String name) { final GenericDescriptor result = @@ -1427,7 +1448,7 @@ public final class Descriptors { /** * Get the method's fully-qualified name. - * @see Descriptor#getFullName() + * @see Descriptors.Descriptor#getFullName() */ public String getFullName() { return fullName; } @@ -1475,7 +1496,8 @@ public final class Descriptors { private void crossLink() throws DescriptorValidationException { final GenericDescriptor input = - file.pool.lookupSymbol(proto.getInputType(), this); + file.pool.lookupSymbol(proto.getInputType(), this, + DescriptorPool.SearchFilter.TYPES_ONLY); if (!(input instanceof Descriptor)) { throw new DescriptorValidationException(this, '\"' + proto.getInputType() + "\" is not a message type."); @@ -1483,7 +1505,8 @@ public final class Descriptors { inputType = (Descriptor)input; final GenericDescriptor output = - file.pool.lookupSymbol(proto.getOutputType(), this); + file.pool.lookupSymbol(proto.getOutputType(), this, + DescriptorPool.SearchFilter.TYPES_ONLY); if (!(output instanceof Descriptor)) { throw new DescriptorValidationException(this, '\"' + proto.getOutputType() + "\" is not a message type."); @@ -1535,7 +1558,7 @@ public final class Descriptors { public String getProblemSymbolName() { return name; } /** - * Gets the the protocol message representation of the invalid descriptor. + * Gets the protocol message representation of the invalid descriptor. */ public Message getProblemProto() { return proto; } @@ -1590,14 +1613,22 @@ public final class Descriptors { * descriptors defined in a particular file. */ private static final class DescriptorPool { + + /** Defines what subclass of descriptors to search in the descriptor pool. + */ + enum SearchFilter { + TYPES_ONLY, AGGREGATES_ONLY, ALL_SYMBOLS + } + DescriptorPool(final FileDescriptor[] dependencies) { - this.dependencies = new DescriptorPool[dependencies.length]; + this.dependencies = new HashSet(); - for (int i = 0; i < dependencies.length; i++) { - this.dependencies[i] = dependencies[i].pool; + for (int i = 0; i < dependencies.length; i++) { + this.dependencies.add(dependencies[i]); + importPublicDependencies(dependencies[i]); } - for (final FileDescriptor dependency : dependencies) { + for (final FileDescriptor dependency : this.dependencies) { try { addPackage(dependency.getPackage(), dependency); } catch (DescriptorValidationException e) { @@ -1609,7 +1640,16 @@ public final class Descriptors { } } - private final DescriptorPool[] dependencies; + /** Find and put public dependencies of the file into dependencies set.*/ + private void importPublicDependencies(final FileDescriptor file) { + for (FileDescriptor dependency : file.getPublicDependencies()) { + if (dependencies.add(dependency)) { + importPublicDependencies(dependency); + } + } + } + + private final Set dependencies; private final Map descriptorsByName = new HashMap(); @@ -1620,39 +1660,81 @@ public final class Descriptors { /** Find a generic descriptor by fully-qualified name. */ GenericDescriptor findSymbol(final String fullName) { + return findSymbol(fullName, SearchFilter.ALL_SYMBOLS); + } + + /** Find a descriptor by fully-qualified name and given option to only + * search valid field type descriptors. + */ + GenericDescriptor findSymbol(final String fullName, + final SearchFilter filter) { GenericDescriptor result = descriptorsByName.get(fullName); if (result != null) { - return result; + if ((filter==SearchFilter.ALL_SYMBOLS) || + ((filter==SearchFilter.TYPES_ONLY) && isType(result)) || + ((filter==SearchFilter.AGGREGATES_ONLY) && isAggregate(result))) { + return result; + } } - for (final DescriptorPool dependency : dependencies) { - result = dependency.descriptorsByName.get(fullName); + for (final FileDescriptor dependency : dependencies) { + result = dependency.pool.descriptorsByName.get(fullName); if (result != null) { - return result; + if ((filter==SearchFilter.ALL_SYMBOLS) || + ((filter==SearchFilter.TYPES_ONLY) && isType(result)) || + ((filter==SearchFilter.AGGREGATES_ONLY) && isAggregate(result))) { + return result; + } } } return null; } + /** Checks if the descriptor is a valid type for a message field. */ + boolean isType(GenericDescriptor descriptor) { + return (descriptor instanceof Descriptor) || + (descriptor instanceof EnumDescriptor); + } + + /** Checks if the descriptor is a valid namespace type. */ + boolean isAggregate(GenericDescriptor descriptor) { + return (descriptor instanceof Descriptor) || + (descriptor instanceof EnumDescriptor) || + (descriptor instanceof PackageDescriptor) || + (descriptor instanceof ServiceDescriptor); + } + /** - * Look up a descriptor by name, relative to some other descriptor. + * Look up a type descriptor by name, relative to some other descriptor. * The name may be fully-qualified (with a leading '.'), * partially-qualified, or unqualified. C++-like name lookup semantics * are used to search for the matching descriptor. */ GenericDescriptor lookupSymbol(final String name, - final GenericDescriptor relativeTo) + final GenericDescriptor relativeTo, + final DescriptorPool.SearchFilter filter) throws DescriptorValidationException { // TODO(kenton): This could be optimized in a number of ways. GenericDescriptor result; if (name.startsWith(".")) { // Fully-qualified name. - result = findSymbol(name.substring(1)); + result = findSymbol(name.substring(1), filter); } else { // If "name" is a compound identifier, we want to search for the // first component of it, then search within it for the rest. + // If name is something like "Foo.Bar.baz", and symbols named "Foo" are + // defined in multiple parent scopes, we only want to find "Bar.baz" in + // the innermost one. E.g., the following should produce an error: + // message Bar { message Baz {} } + // message Foo { + // message Bar { + // } + // optional Bar.Baz baz = 1; + // } + // So, we look for just "Foo" first, then look for "Bar.baz" within it + // if found. final int firstPartLength = name.indexOf('.'); final String firstPart; if (firstPartLength == -1) { @@ -1670,14 +1752,15 @@ public final class Descriptors { // Chop off the last component of the scope. final int dotpos = scopeToTry.lastIndexOf("."); if (dotpos == -1) { - result = findSymbol(name); + result = findSymbol(name, filter); break; } else { scopeToTry.setLength(dotpos + 1); - // Append firstPart and try to find. + // Append firstPart and try to find scopeToTry.append(firstPart); - result = findSymbol(scopeToTry.toString()); + result = findSymbol(scopeToTry.toString(), + DescriptorPool.SearchFilter.AGGREGATES_ONLY); if (result != null) { if (firstPartLength != -1) { @@ -1686,7 +1769,7 @@ public final class Descriptors { // searching parent scopes. scopeToTry.setLength(dotpos + 1); scopeToTry.append(name); - result = findSymbol(scopeToTry.toString()); + result = findSymbol(scopeToTry.toString(), filter); } break; } @@ -1817,7 +1900,7 @@ public final class Descriptors { /** * Adds a field to the fieldsByNumber table. Throws an exception if a - * field with hte same containing type and number already exists. + * field with the same containing type and number already exists. */ void addFieldByNumber(final FieldDescriptor field) throws DescriptorValidationException { diff --git a/java/src/main/java/com/google/protobuf/DynamicMessage.java b/java/src/main/java/com/google/protobuf/DynamicMessage.java index c106b662..c0c9fc94 100644 --- a/java/src/main/java/com/google/protobuf/DynamicMessage.java +++ b/java/src/main/java/com/google/protobuf/DynamicMessage.java @@ -35,6 +35,7 @@ import com.google.protobuf.Descriptors.FieldDescriptor; import java.io.InputStream; import java.io.IOException; +import java.util.Collections; import java.util.Map; /** @@ -160,7 +161,9 @@ public final class DynamicMessage extends AbstractMessage { verifyContainingType(field); Object result = fields.getField(field); if (result == null) { - if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { + if (field.isRepeated()) { + result = Collections.emptyList(); + } else if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { result = getDefaultInstance(field.getMessageType()); } else { result = field.getDefaultValue(); @@ -198,10 +201,12 @@ public final class DynamicMessage extends AbstractMessage { return fields.isInitialized(); } + @Override public boolean isInitialized() { return isInitialized(type, fields); } + @Override public void writeTo(CodedOutputStream output) throws IOException { if (type.getOptions().getMessageSetWireFormat()) { fields.writeMessageSetTo(output); @@ -212,6 +217,7 @@ public final class DynamicMessage extends AbstractMessage { } } + @Override public int getSerializedSize() { int size = memoizedSize; if (size != -1) return size; @@ -236,6 +242,26 @@ public final class DynamicMessage extends AbstractMessage { return newBuilderForType().mergeFrom(this); } + public Parser getParserForType() { + return new AbstractParser() { + public DynamicMessage parsePartialFrom( + CodedInputStream input, + ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + Builder builder = newBuilder(type); + try { + builder.mergeFrom(input, extensionRegistry); + } catch (InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(builder.buildPartial()); + } catch (IOException e) { + throw new InvalidProtocolBufferException(e.getMessage()) + .setUnfinishedMessage(builder.buildPartial()); + } + return builder.buildPartial(); + } + }; + } + /** Verifies that the field is a field of this message. */ private void verifyContainingType(FieldDescriptor field) { if (field.getContainingType() != type) { @@ -264,14 +290,18 @@ public final class DynamicMessage extends AbstractMessage { // --------------------------------------------------------------- // Implementation of Message.Builder interface. + @Override public Builder clear() { - if (fields == null) { - throw new IllegalStateException("Cannot call clear() after build()."); + if (fields.isImmutable()) { + fields = FieldSet.newFieldSet(); + } else { + fields.clear(); } - fields.clear(); + unknownFields = UnknownFieldSet.getDefaultInstance(); return this; } + @Override public Builder mergeFrom(Message other) { if (other instanceof DynamicMessage) { // This should be somewhat faster than calling super.mergeFrom(). @@ -280,6 +310,7 @@ public final class DynamicMessage extends AbstractMessage { throw new IllegalArgumentException( "mergeFrom(Message) can only merge messages of the same type."); } + ensureIsMutable(); fields.mergeFrom(otherDynamicMessage.fields); mergeUnknownFields(otherDynamicMessage.unknownFields); return this; @@ -289,8 +320,7 @@ public final class DynamicMessage extends AbstractMessage { } public DynamicMessage build() { - // If fields == null, we'll throw an appropriate exception later. - if (fields != null && !isInitialized()) { + if (!isInitialized()) { throw newUninitializedMessageException( new DynamicMessage(type, fields, unknownFields)); } @@ -312,21 +342,17 @@ public final class DynamicMessage extends AbstractMessage { } public DynamicMessage buildPartial() { - if (fields == null) { - throw new IllegalStateException( - "build() has already been called on this Builder."); - } fields.makeImmutable(); DynamicMessage result = new DynamicMessage(type, fields, unknownFields); - fields = null; - unknownFields = null; return result; } + @Override public Builder clone() { Builder result = new Builder(type); result.fields.mergeFrom(fields); + result.mergeUnknownFields(unknownFields); return result; } @@ -377,12 +403,14 @@ public final class DynamicMessage extends AbstractMessage { public Builder setField(FieldDescriptor field, Object value) { verifyContainingType(field); + ensureIsMutable(); fields.setField(field, value); return this; } public Builder clearField(FieldDescriptor field) { verifyContainingType(field); + ensureIsMutable(); fields.clearField(field); return this; } @@ -400,12 +428,14 @@ public final class DynamicMessage extends AbstractMessage { public Builder setRepeatedField(FieldDescriptor field, int index, Object value) { verifyContainingType(field); + ensureIsMutable(); fields.setRepeatedField(field, index, value); return this; } public Builder addRepeatedField(FieldDescriptor field, Object value) { verifyContainingType(field); + ensureIsMutable(); fields.addRepeatedField(field, value); return this; } @@ -419,6 +449,7 @@ public final class DynamicMessage extends AbstractMessage { return this; } + @Override public Builder mergeUnknownFields(UnknownFieldSet unknownFields) { this.unknownFields = UnknownFieldSet.newBuilder(this.unknownFields) @@ -434,5 +465,18 @@ public final class DynamicMessage extends AbstractMessage { "FieldDescriptor does not match message type."); } } + + private void ensureIsMutable() { + if (fields.isImmutable()) { + fields = fields.clone(); + } + } + + @Override + public com.google.protobuf.Message.Builder getFieldBuilder(FieldDescriptor field) { + // TODO(xiangl): need implementation for dynamic message + throw new UnsupportedOperationException( + "getFieldBuilder() called on a dynamic message type."); + } } } diff --git a/java/src/main/java/com/google/protobuf/ExtensionRegistryLite.java b/java/src/main/java/com/google/protobuf/ExtensionRegistryLite.java index d5288dd8..1e1289d0 100644 --- a/java/src/main/java/com/google/protobuf/ExtensionRegistryLite.java +++ b/java/src/main/java/com/google/protobuf/ExtensionRegistryLite.java @@ -43,7 +43,7 @@ import java.util.Map; * make sense to mix the two, since if you have any regular types in your * program, you then require the full runtime and lose all the benefits of * the lite runtime, so you might as well make all your types be regular types. - * However, in some cases (e.g. when depending on multiple third-patry libraries + * However, in some cases (e.g. when depending on multiple third-party libraries * where one uses lite types and one uses regular), you may find yourself * wanting to mix the two. In this case things get more complicated. *

@@ -71,6 +71,22 @@ import java.util.Map; * @author kenton@google.com Kenton Varda */ public class ExtensionRegistryLite { + + // Set true to enable lazy parsing feature for MessageSet. + // + // TODO(xiangl): Now we use a global flag to control whether enable lazy + // parsing feature for MessageSet, which may be too crude for some + // applications. Need to support this feature on smaller granularity. + private static volatile boolean eagerlyParseMessageSets = false; + + public static boolean isEagerlyParseMessageSets() { + return eagerlyParseMessageSets; + } + + public static void setEagerlyParseMessageSets(boolean isEagerlyParse) { + eagerlyParseMessageSets = isEagerlyParse; + } + /** Construct a new, empty instance. */ public static ExtensionRegistryLite newInstance() { return new ExtensionRegistryLite(); diff --git a/java/src/main/java/com/google/protobuf/FieldSet.java b/java/src/main/java/com/google/protobuf/FieldSet.java index a85dbaa6..2663694f 100644 --- a/java/src/main/java/com/google/protobuf/FieldSet.java +++ b/java/src/main/java/com/google/protobuf/FieldSet.java @@ -30,12 +30,14 @@ package com.google.protobuf; +import com.google.protobuf.LazyField.LazyIterator; + +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.io.IOException; /** * A class which represents an arbitrary set of fields of some message type. @@ -68,6 +70,7 @@ final class FieldSet fields; private boolean isImmutable; + private boolean hasLazyField = false; /** Construct a new FieldSet. */ private FieldSet() { @@ -95,7 +98,7 @@ final class FieldSet emptySet() { return DEFAULT_INSTANCE; } - @SuppressWarnings("unchecked") + @SuppressWarnings("rawtypes") private static final FieldSet DEFAULT_INSTANCE = new FieldSet(true); /** Make this FieldSet immutable from this point forward. */ @@ -109,7 +112,7 @@ final class FieldSet getAllFields() { + if (hasLazyField) { + SmallSortedMap result = + SmallSortedMap.newFieldMap(16); + for (int i = 0; i < fields.getNumArrayEntries(); i++) { + cloneFieldEntry(result, fields.getArrayEntryAt(i)); + } + for (Map.Entry entry : + fields.getOverflowEntries()) { + cloneFieldEntry(result, entry); + } + if (fields.isImmutable()) { + result.makeImmutable(); + } + return result; + } return fields.isImmutable() ? fields : Collections.unmodifiableMap(fields); } + private void cloneFieldEntry(Map map, + Map.Entry entry) { + FieldDescriptorType key = entry.getKey(); + Object value = entry.getValue(); + if (value instanceof LazyField) { + map.put(key, ((LazyField) value).getValue()); + } else { + map.put(key, value); + } + } + /** * Get an iterator to the field map. This iterator should not be leaked out - * of the protobuf library as it is not protected from mutation when - * fields is not immutable. + * of the protobuf library as it is not protected from mutation when fields + * is not immutable. */ public Iterator> iterator() { + if (hasLazyField) { + return new LazyIterator( + fields.entrySet().iterator()); + } return fields.entrySet().iterator(); } @@ -185,14 +220,18 @@ final class FieldSet) list).set(index, value); } /** @@ -298,13 +343,13 @@ final class FieldSet list; if (existingValue == null) { - list = new ArrayList(); + list = new ArrayList(); fields.put(descriptor, list); } else { - list = (List) existingValue; + list = (List) existingValue; } list.add(value); @@ -338,7 +383,8 @@ final class FieldSet other) { for (int i = 0; i < other.fields.getNumArrayEntries(); i++) { @@ -428,14 +483,17 @@ final class FieldSet entry) { final FieldDescriptorType descriptor = entry.getKey(); - final Object otherValue = entry.getValue(); + Object otherValue = entry.getValue(); + if (otherValue instanceof LazyField) { + otherValue = ((LazyField) otherValue).getValue(); + } if (descriptor.isRepeated()) { - Object value = fields.get(descriptor); + Object value = getField(descriptor); if (value == null) { // Our list is empty, but we still need to make a defensive copy of // the other list since we don't know if the other FieldSet is still @@ -446,7 +504,7 @@ final class FieldSet entry) { final FieldDescriptorType descriptor = entry.getKey(); - if (descriptor.getLiteJavaType() == WireFormat.JavaType.MESSAGE && - !descriptor.isRepeated() && !descriptor.isPacked()) { - return CodedOutputStream.computeMessageSetExtensionSize( - entry.getKey().getNumber(), (MessageLite) entry.getValue()); + Object value = entry.getValue(); + if (descriptor.getLiteJavaType() == WireFormat.JavaType.MESSAGE + && !descriptor.isRepeated() && !descriptor.isPacked()) { + if (value instanceof LazyField) { + return CodedOutputStream.computeLazyFieldMessageSetExtensionSize( + entry.getKey().getNumber(), (LazyField) value); + } else { + return CodedOutputStream.computeMessageSetExtensionSize( + entry.getKey().getNumber(), (MessageLite) value); + } } else { - return computeFieldSize(descriptor, entry.getValue()); + return computeFieldSize(descriptor, value); } } @@ -741,7 +808,6 @@ final class FieldSet builder) { - this.unknownFields = builder.getUnknownFields(); + } + + public Parser getParserForType() { + throw new UnsupportedOperationException( + "This is supposed to be overridden by subclasses."); } /** @@ -175,8 +176,28 @@ public abstract class GeneratedMessage extends AbstractMessage } //@Override (Java 1.6 override semantics, but we must support 1.5) - public final UnknownFieldSet getUnknownFields() { - return unknownFields; + public UnknownFieldSet getUnknownFields() { + throw new UnsupportedOperationException( + "This is supposed to be overridden by subclasses."); + } + + /** + * Called by subclasses to parse an unknown field. + * @return {@code true} unless the tag is an end-group tag. + */ + protected boolean parseUnknownField( + CodedInputStream input, + UnknownFieldSet.Builder unknownFields, + ExtensionRegistryLite extensionRegistry, + int tag) throws IOException { + return unknownFields.mergeFieldFrom(tag, input); + } + + /** + * Used by parsing constructors in generated classes. + */ + protected void makeExtensionsImmutable() { + // Noop for messages without extensions. } protected abstract Message.Builder newBuilderForType(BuilderParent parent); @@ -318,6 +339,11 @@ public abstract class GeneratedMessage extends AbstractMessage return internalGetFieldAccessorTable().getField(field).newBuilder(); } + //@Override (Java 1.6 override semantics, but we must support 1.5) + public Message.Builder getFieldBuilder(final FieldDescriptor field) { + return internalGetFieldAccessorTable().getField(field).getBuilder(this); + } + //@Override (Java 1.6 override semantics, but we must support 1.5) public boolean hasField(final FieldDescriptor field) { return internalGetFieldAccessorTable().getField(field).has(this); @@ -626,6 +652,25 @@ public abstract class GeneratedMessage extends AbstractMessage return super.isInitialized() && extensionsAreInitialized(); } + @Override + protected boolean parseUnknownField( + CodedInputStream input, + UnknownFieldSet.Builder unknownFields, + ExtensionRegistryLite extensionRegistry, + int tag) throws IOException { + return AbstractMessage.Builder.mergeFieldFrom( + input, unknownFields, extensionRegistry, getDescriptorForType(), + null, extensions, tag); + } + + /** + * Used by parsing constructors in generated classes. + */ + @Override + protected void makeExtensionsImmutable() { + extensions.makeImmutable(); + } + /** * Used by subclasses to serialize extensions. Extension ranges may be * interleaved with field numbers, but we must write them in canonical @@ -655,9 +700,21 @@ public abstract class GeneratedMessage extends AbstractMessage if (messageSetWireFormat && descriptor.getLiteJavaType() == WireFormat.JavaType.MESSAGE && !descriptor.isRepeated()) { - output.writeMessageSetExtension(descriptor.getNumber(), - (Message) next.getValue()); + if (next instanceof LazyField.LazyEntry) { + output.writeRawMessageSetExtension(descriptor.getNumber(), + ((LazyField.LazyEntry) next).getField().toByteString()); + } else { + output.writeMessageSetExtension(descriptor.getNumber(), + (Message) next.getValue()); + } } else { + // TODO(xiangl): Taken care of following code, it may cause + // problem when we use LazyField for normal fields/extensions. + // Due to the optional field can be duplicated at the end of + // serialized bytes, which will make the serialized size change + // after lazy field parsed. So when we use LazyField globally, + // we need to change the following write method to write cached + // bytes directly rather than write the parsed message. FieldSet.writeField(descriptor, next.getValue(), output); } if (iter.hasNext()) { @@ -974,7 +1031,8 @@ public abstract class GeneratedMessage extends AbstractMessage final ExtensionRegistryLite extensionRegistry, final int tag) throws IOException { return AbstractMessage.Builder.mergeFieldFrom( - input, unknownFields, extensionRegistry, this, tag); + input, unknownFields, extensionRegistry, getDescriptorForType(), + this, null, tag); } // --------------------------------------------------------------- @@ -1405,39 +1463,72 @@ public abstract class GeneratedMessage extends AbstractMessage final String[] camelCaseNames, final Class messageClass, final Class builderClass) { + this(descriptor, camelCaseNames); + ensureFieldAccessorsInitialized(messageClass, builderClass); + } + + /** + * Construct a FieldAccessorTable for a particular message class without + * initializing FieldAccessors. + */ + public FieldAccessorTable( + final Descriptor descriptor, + final String[] camelCaseNames) { this.descriptor = descriptor; + this.camelCaseNames = camelCaseNames; fields = new FieldAccessor[descriptor.getFields().size()]; + initialized = false; + } - for (int i = 0; i < fields.length; i++) { - final FieldDescriptor field = descriptor.getFields().get(i); - if (field.isRepeated()) { - if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { - fields[i] = new RepeatedMessageFieldAccessor( - field, camelCaseNames[i], messageClass, builderClass); - } else if (field.getJavaType() == FieldDescriptor.JavaType.ENUM) { - fields[i] = new RepeatedEnumFieldAccessor( - field, camelCaseNames[i], messageClass, builderClass); - } else { - fields[i] = new RepeatedFieldAccessor( - field, camelCaseNames[i], messageClass, builderClass); - } - } else { - if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { - fields[i] = new SingularMessageFieldAccessor( - field, camelCaseNames[i], messageClass, builderClass); - } else if (field.getJavaType() == FieldDescriptor.JavaType.ENUM) { - fields[i] = new SingularEnumFieldAccessor( - field, camelCaseNames[i], messageClass, builderClass); + /** + * Ensures the field accessors are initialized. This method is thread-safe. + * + * @param messageClass The message type. + * @param builderClass The builder type. + * @return this + */ + public FieldAccessorTable ensureFieldAccessorsInitialized( + Class messageClass, + Class builderClass) { + if (initialized) { return this; } + synchronized (this) { + if (initialized) { return this; } + for (int i = 0; i < fields.length; i++) { + FieldDescriptor field = descriptor.getFields().get(i); + if (field.isRepeated()) { + if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { + fields[i] = new RepeatedMessageFieldAccessor( + field, camelCaseNames[i], messageClass, builderClass); + } else if (field.getJavaType() == FieldDescriptor.JavaType.ENUM) { + fields[i] = new RepeatedEnumFieldAccessor( + field, camelCaseNames[i], messageClass, builderClass); + } else { + fields[i] = new RepeatedFieldAccessor( + field, camelCaseNames[i], messageClass, builderClass); + } } else { - fields[i] = new SingularFieldAccessor( - field, camelCaseNames[i], messageClass, builderClass); + if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { + fields[i] = new SingularMessageFieldAccessor( + field, camelCaseNames[i], messageClass, builderClass); + } else if (field.getJavaType() == FieldDescriptor.JavaType.ENUM) { + fields[i] = new SingularEnumFieldAccessor( + field, camelCaseNames[i], messageClass, builderClass); + } else { + fields[i] = new SingularFieldAccessor( + field, camelCaseNames[i], messageClass, builderClass); + } } } + initialized = true; + camelCaseNames = null; + return this; } } private final Descriptor descriptor; private final FieldAccessor[] fields; + private String[] camelCaseNames; + private volatile boolean initialized; /** Get the FieldAccessor for a particular field. */ private FieldAccessor getField(final FieldDescriptor field) { @@ -1472,6 +1563,7 @@ public abstract class GeneratedMessage extends AbstractMessage int getRepeatedCount(GeneratedMessage.Builder builder); void clear(Builder builder); Message.Builder newBuilder(); + Message.Builder getBuilder(GeneratedMessage.Builder builder); } // --------------------------------------------------------------- @@ -1551,6 +1643,10 @@ public abstract class GeneratedMessage extends AbstractMessage throw new UnsupportedOperationException( "newBuilderForField() called on a non-Message type."); } + public Message.Builder getBuilder(GeneratedMessage.Builder builder) { + throw new UnsupportedOperationException( + "getFieldBuilder() called on a non-Message type."); + } } private static class RepeatedFieldAccessor implements FieldAccessor { @@ -1573,8 +1669,6 @@ public abstract class GeneratedMessage extends AbstractMessage "get" + camelCaseName + "List"); getMethodBuilder = getMethodOrDie(builderClass, "get" + camelCaseName + "List"); - - getRepeatedMethod = getMethodOrDie(messageClass, "get" + camelCaseName, Integer.TYPE); getRepeatedMethodBuilder = @@ -1625,11 +1719,11 @@ public abstract class GeneratedMessage extends AbstractMessage } public boolean has(final GeneratedMessage message) { throw new UnsupportedOperationException( - "hasField() called on a singular field."); + "hasField() called on a repeated field."); } public boolean has(GeneratedMessage.Builder builder) { throw new UnsupportedOperationException( - "hasField() called on a singular field."); + "hasField() called on a repeated field."); } public int getRepeatedCount(final GeneratedMessage message) { return (Integer) invokeOrDie(getCountMethod, message); @@ -1644,6 +1738,10 @@ public abstract class GeneratedMessage extends AbstractMessage throw new UnsupportedOperationException( "newBuilderForField() called on a non-Message type."); } + public Message.Builder getBuilder(GeneratedMessage.Builder builder) { + throw new UnsupportedOperationException( + "getFieldBuilder() called on a non-Message type."); + } } // --------------------------------------------------------------- @@ -1753,9 +1851,12 @@ public abstract class GeneratedMessage extends AbstractMessage super(descriptor, camelCaseName, messageClass, builderClass); newBuilderMethod = getMethodOrDie(type, "newBuilder"); + getBuilderMethodBuilder = + getMethodOrDie(builderClass, "get" + camelCaseName + "Builder"); } private final Method newBuilderMethod; + private final Method getBuilderMethodBuilder; private Object coerceType(final Object value) { if (type.isInstance(value)) { @@ -1766,7 +1867,7 @@ public abstract class GeneratedMessage extends AbstractMessage // DynamicMessage -- we should accept it. In this case we can make // a copy of the message. return ((Message.Builder) invokeOrDie(newBuilderMethod, null)) - .mergeFrom((Message) value).build(); + .mergeFrom((Message) value).buildPartial(); } } @@ -1778,6 +1879,10 @@ public abstract class GeneratedMessage extends AbstractMessage public Message.Builder newBuilder() { return (Message.Builder) invokeOrDie(newBuilderMethod, null); } + @Override + public Message.Builder getBuilder(GeneratedMessage.Builder builder) { + return (Message.Builder) invokeOrDie(getBuilderMethodBuilder, builder); + } } private static final class RepeatedMessageFieldAccessor @@ -1825,7 +1930,7 @@ public abstract class GeneratedMessage extends AbstractMessage /** * Replaces this object in the output stream with a serialized form. * Part of Java's serialization magic. Generated sub-classes must override - * this method by calling return super.writeReplace(); + * this method by calling {@code return super.writeReplace();} * @return a SerializedForm of this message */ protected Object writeReplace() throws ObjectStreamException { diff --git a/java/src/main/java/com/google/protobuf/GeneratedMessageLite.java b/java/src/main/java/com/google/protobuf/GeneratedMessageLite.java index 1813e9b3..437e3412 100644 --- a/java/src/main/java/com/google/protobuf/GeneratedMessageLite.java +++ b/java/src/main/java/com/google/protobuf/GeneratedMessageLite.java @@ -55,6 +55,29 @@ public abstract class GeneratedMessageLite extends AbstractMessageLite protected GeneratedMessageLite(Builder builder) { } + public Parser getParserForType() { + throw new UnsupportedOperationException( + "This is supposed to be overridden by subclasses."); + } + + /** + * Called by subclasses to parse an unknown field. + * @return {@code true} unless the tag is an end-group tag. + */ + protected boolean parseUnknownField( + CodedInputStream input, + ExtensionRegistryLite extensionRegistry, + int tag) throws IOException { + return input.skipField(tag); + } + + /** + * Used by parsing constructors in generated classes. + */ + protected void makeExtensionsImmutable() { + // Noop for messages without extensions. + } + @SuppressWarnings("unchecked") public abstract static class Builder @@ -86,9 +109,9 @@ public abstract class GeneratedMessageLite extends AbstractMessageLite * @return {@code true} unless the tag is an end-group tag. */ protected boolean parseUnknownField( - final CodedInputStream input, - final ExtensionRegistryLite extensionRegistry, - final int tag) throws IOException { + CodedInputStream input, + ExtensionRegistryLite extensionRegistry, + int tag) throws IOException { return input.skipField(tag); } } @@ -193,6 +216,31 @@ public abstract class GeneratedMessageLite extends AbstractMessageLite return extensions.isInitialized(); } + /** + * Called by subclasses to parse an unknown field or an extension. + * @return {@code true} unless the tag is an end-group tag. + */ + @Override + protected boolean parseUnknownField( + CodedInputStream input, + ExtensionRegistryLite extensionRegistry, + int tag) throws IOException { + return GeneratedMessageLite.parseUnknownField( + extensions, + getDefaultInstanceForType(), + input, + extensionRegistry, + tag); + } + + /** + * Used by parsing constructors in generated classes. + */ + @Override + protected void makeExtensionsImmutable() { + extensions.makeImmutable(); + } + /** * Used by subclasses to serialize extensions. Extension ranges may be * interleaved with field numbers, but we must write them in canonical @@ -400,121 +448,139 @@ public abstract class GeneratedMessageLite extends AbstractMessageLite */ @Override protected boolean parseUnknownField( - final CodedInputStream input, - final ExtensionRegistryLite extensionRegistry, - final int tag) throws IOException { - final int wireType = WireFormat.getTagWireType(tag); - final int fieldNumber = WireFormat.getTagFieldNumber(tag); - - final GeneratedExtension extension = - extensionRegistry.findLiteExtensionByNumber( - getDefaultInstanceForType(), fieldNumber); - - boolean unknown = false; - boolean packed = false; - if (extension == null) { - unknown = true; // Unknown field. - } else if (wireType == FieldSet.getWireFormatForFieldType( - extension.descriptor.getLiteType(), - false /* isPacked */)) { - packed = false; // Normal, unpacked value. - } else if (extension.descriptor.isRepeated && - extension.descriptor.type.isPackable() && - wireType == FieldSet.getWireFormatForFieldType( - extension.descriptor.getLiteType(), - true /* isPacked */)) { - packed = true; // Packed value. - } else { - unknown = true; // Wrong wire type. - } + CodedInputStream input, + ExtensionRegistryLite extensionRegistry, + int tag) throws IOException { + ensureExtensionsIsMutable(); + return GeneratedMessageLite.parseUnknownField( + extensions, + getDefaultInstanceForType(), + input, + extensionRegistry, + tag); + } - if (unknown) { // Unknown field or wrong wire type. Skip. - return input.skipField(tag); - } + protected final void mergeExtensionFields(final MessageType other) { + ensureExtensionsIsMutable(); + extensions.mergeFrom(((ExtendableMessage) other).extensions); + } + } - if (packed) { - final int length = input.readRawVarint32(); - final int limit = input.pushLimit(length); - if (extension.descriptor.getLiteType() == WireFormat.FieldType.ENUM) { - while (input.getBytesUntilLimit() > 0) { - final int rawValue = input.readEnum(); - final Object value = - extension.descriptor.getEnumType().findValueByNumber(rawValue); - if (value == null) { - // If the number isn't recognized as a valid value for this - // enum, drop it (don't even add it to unknownFields). - return true; - } - ensureExtensionsIsMutable(); - extensions.addRepeatedField(extension.descriptor, value); - } - } else { - while (input.getBytesUntilLimit() > 0) { - final Object value = - FieldSet.readPrimitiveField(input, - extension.descriptor.getLiteType()); - ensureExtensionsIsMutable(); - extensions.addRepeatedField(extension.descriptor, value); + // ----------------------------------------------------------------- + + /** + * Parse an unknown field or an extension. + * @return {@code true} unless the tag is an end-group tag. + */ + private static + boolean parseUnknownField( + FieldSet extensions, + MessageType defaultInstance, + CodedInputStream input, + ExtensionRegistryLite extensionRegistry, + int tag) throws IOException { + int wireType = WireFormat.getTagWireType(tag); + int fieldNumber = WireFormat.getTagFieldNumber(tag); + + GeneratedExtension extension = + extensionRegistry.findLiteExtensionByNumber( + defaultInstance, fieldNumber); + + boolean unknown = false; + boolean packed = false; + if (extension == null) { + unknown = true; // Unknown field. + } else if (wireType == FieldSet.getWireFormatForFieldType( + extension.descriptor.getLiteType(), + false /* isPacked */)) { + packed = false; // Normal, unpacked value. + } else if (extension.descriptor.isRepeated && + extension.descriptor.type.isPackable() && + wireType == FieldSet.getWireFormatForFieldType( + extension.descriptor.getLiteType(), + true /* isPacked */)) { + packed = true; // Packed value. + } else { + unknown = true; // Wrong wire type. + } + + if (unknown) { // Unknown field or wrong wire type. Skip. + return input.skipField(tag); + } + + if (packed) { + int length = input.readRawVarint32(); + int limit = input.pushLimit(length); + if (extension.descriptor.getLiteType() == WireFormat.FieldType.ENUM) { + while (input.getBytesUntilLimit() > 0) { + int rawValue = input.readEnum(); + Object value = + extension.descriptor.getEnumType().findValueByNumber(rawValue); + if (value == null) { + // If the number isn't recognized as a valid value for this + // enum, drop it (don't even add it to unknownFields). + return true; } + extensions.addRepeatedField(extension.descriptor, value); } - input.popLimit(limit); } else { - final Object value; - switch (extension.descriptor.getLiteJavaType()) { - case MESSAGE: { - MessageLite.Builder subBuilder = null; - if (!extension.descriptor.isRepeated()) { - MessageLite existingValue = - (MessageLite) extensions.getField(extension.descriptor); - if (existingValue != null) { - subBuilder = existingValue.toBuilder(); - } - } - if (subBuilder == null) { - subBuilder = extension.messageDefaultInstance.newBuilderForType(); - } - if (extension.descriptor.getLiteType() == - WireFormat.FieldType.GROUP) { - input.readGroup(extension.getNumber(), - subBuilder, extensionRegistry); - } else { - input.readMessage(subBuilder, extensionRegistry); + while (input.getBytesUntilLimit() > 0) { + Object value = + FieldSet.readPrimitiveField(input, + extension.descriptor.getLiteType()); + extensions.addRepeatedField(extension.descriptor, value); + } + } + input.popLimit(limit); + } else { + Object value; + switch (extension.descriptor.getLiteJavaType()) { + case MESSAGE: { + MessageLite.Builder subBuilder = null; + if (!extension.descriptor.isRepeated()) { + MessageLite existingValue = + (MessageLite) extensions.getField(extension.descriptor); + if (existingValue != null) { + subBuilder = existingValue.toBuilder(); } - value = subBuilder.build(); - break; } - case ENUM: - final int rawValue = input.readEnum(); - value = extension.descriptor.getEnumType() - .findValueByNumber(rawValue); - // If the number isn't recognized as a valid value for this enum, - // drop it. - if (value == null) { - return true; - } - break; - default: - value = FieldSet.readPrimitiveField(input, - extension.descriptor.getLiteType()); - break; - } - - if (extension.descriptor.isRepeated()) { - ensureExtensionsIsMutable(); - extensions.addRepeatedField(extension.descriptor, value); - } else { - ensureExtensionsIsMutable(); - extensions.setField(extension.descriptor, value); + if (subBuilder == null) { + subBuilder = extension.messageDefaultInstance.newBuilderForType(); + } + if (extension.descriptor.getLiteType() == + WireFormat.FieldType.GROUP) { + input.readGroup(extension.getNumber(), + subBuilder, extensionRegistry); + } else { + input.readMessage(subBuilder, extensionRegistry); + } + value = subBuilder.build(); + break; } + case ENUM: + int rawValue = input.readEnum(); + value = extension.descriptor.getEnumType() + .findValueByNumber(rawValue); + // If the number isn't recognized as a valid value for this enum, + // drop it. + if (value == null) { + return true; + } + break; + default: + value = FieldSet.readPrimitiveField(input, + extension.descriptor.getLiteType()); + break; } - return true; + if (extension.descriptor.isRepeated()) { + extensions.addRepeatedField(extension.descriptor, value); + } else { + extensions.setField(extension.descriptor, value); + } } - protected final void mergeExtensionFields(final MessageType other) { - ensureExtensionsIsMutable(); - extensions.mergeFrom(((ExtendableMessage) other).extensions); - } + return true; } // ----------------------------------------------------------------- @@ -722,7 +788,7 @@ public abstract class GeneratedMessageLite extends AbstractMessageLite /** * Replaces this object in the output stream with a serialized form. * Part of Java's serialization magic. Generated sub-classes must override - * this method by calling return super.writeReplace(); + * this method by calling {@code return super.writeReplace();} * @return a SerializedForm of this message */ protected Object writeReplace() throws ObjectStreamException { diff --git a/java/src/main/java/com/google/protobuf/Internal.java b/java/src/main/java/com/google/protobuf/Internal.java index 05eab57a..81af2583 100644 --- a/java/src/main/java/com/google/protobuf/Internal.java +++ b/java/src/main/java/com/google/protobuf/Internal.java @@ -103,85 +103,32 @@ public class Internal { * Helper called by generated code to determine if a byte array is a valid * UTF-8 encoded string such that the original bytes can be converted to * a String object and then back to a byte array round tripping the bytes - * without loss. - *

- * This is inspired by UTF_8.java in sun.nio.cs. + * without loss. More precisely, returns {@code true} whenever: + * 

   {@code
+   * Arrays.equals(byteString.toByteArray(),
+   *     new String(byteString.toByteArray(), "UTF-8").getBytes("UTF-8"))
+   * }
+ * + *

This method rejects "overlong" byte sequences, as well as + * 3-byte sequences that would map to a surrogate character, in + * accordance with the restricted definition of UTF-8 introduced in + * Unicode 3.1. Note that the UTF-8 decoder included in Oracle's + * JDK has been modified to also reject "overlong" byte sequences, + * but currently (2011) still accepts 3-byte surrogate character + * byte sequences. + * + *

See the Unicode Standard,
+ * Table 3-6. UTF-8 Bit Distribution,
+ * Table 3-7. Well Formed UTF-8 Byte Sequences. + * + *

As of 2011-02, this method simply returns the result of {@link + * ByteString#isValidUtf8()}. Calling that method directly is preferred. * * @param byteString the string to check * @return whether the byte array is round trippable */ public static boolean isValidUtf8(ByteString byteString) { - int index = 0; - int size = byteString.size(); - // To avoid the masking, we could change this to use bytes; - // Then X > 0xC2 gets turned into X < -0xC2; X < 0x80 - // gets turned into X >= 0, etc. - - while (index < size) { - int byte1 = byteString.byteAt(index++) & 0xFF; - if (byte1 < 0x80) { - // fast loop for single bytes - continue; - - // we know from this point on that we have 2-4 byte forms - } else if (byte1 < 0xC2 || byte1 > 0xF4) { - // catch illegal first bytes: < C2 or > F4 - return false; - } - if (index >= size) { - // fail if we run out of bytes - return false; - } - int byte2 = byteString.byteAt(index++) & 0xFF; - if (byte2 < 0x80 || byte2 > 0xBF) { - // general trail-byte test - return false; - } - if (byte1 <= 0xDF) { - // two-byte form; general trail-byte test is sufficient - continue; - } - - // we know from this point on that we have 3 or 4 byte forms - if (index >= size) { - // fail if we run out of bytes - return false; - } - int byte3 = byteString.byteAt(index++) & 0xFF; - if (byte3 < 0x80 || byte3 > 0xBF) { - // general trail-byte test - return false; - } - if (byte1 <= 0xEF) { - // three-byte form. Vastly more frequent than four-byte forms - // The following has an extra test, but not worth restructuring - if (byte1 == 0xE0 && byte2 < 0xA0 || - byte1 == 0xED && byte2 > 0x9F) { - // check special cases of byte2 - return false; - } - - } else { - // four-byte form - - if (index >= size) { - // fail if we run out of bytes - return false; - } - int byte4 = byteString.byteAt(index++) & 0xFF; - if (byte4 < 0x80 || byte4 > 0xBF) { - // general trail-byte test - return false; - } - // The following has an extra test, but not worth restructuring - if (byte1 == 0xF0 && byte2 < 0x90 || - byte1 == 0xF4 && byte2 > 0x8F) { - // check special cases of byte2 - return false; - } - } - } - return true; + return byteString.isValidUtf8(); } /** diff --git a/java/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java b/java/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java index 90f7ffbc..72d7ff7d 100644 --- a/java/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java +++ b/java/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java @@ -40,11 +40,32 @@ import java.io.IOException; */ public class InvalidProtocolBufferException extends IOException { private static final long serialVersionUID = -1616151763072450476L; + private MessageLite unfinishedMessage = null; public InvalidProtocolBufferException(final String description) { super(description); } + /** + * Attaches an unfinished message to the exception to support best-effort + * parsing in {@code Parser} interface. + * + * @return this + */ + public InvalidProtocolBufferException setUnfinishedMessage( + MessageLite unfinishedMessage) { + this.unfinishedMessage = unfinishedMessage; + return this; + } + + /** + * Returns the unfinished message attached to the exception, or null if + * no message is attached. + */ + public MessageLite getUnfinishedMessage() { + return unfinishedMessage; + } + static InvalidProtocolBufferException truncatedMessage() { return new InvalidProtocolBufferException( "While parsing a protocol message, the input ended unexpectedly " + diff --git a/java/src/main/java/com/google/protobuf/LazyField.java b/java/src/main/java/com/google/protobuf/LazyField.java new file mode 100644 index 00000000..df9425eb --- /dev/null +++ b/java/src/main/java/com/google/protobuf/LazyField.java @@ -0,0 +1,216 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// http://code.google.com/p/protobuf/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import java.io.IOException; +import java.util.Iterator; +import java.util.Map.Entry; + +/** + * LazyField encapsulates the logic of lazily parsing message fields. It stores + * the message in a ByteString initially and then parse it on-demand. + * + * LazyField is thread-compatible e.g. concurrent read are safe, however, + * synchronizations are needed under read/write situations. + * + * Now LazyField is only used to lazily load MessageSet. + * TODO(xiangl): Use LazyField to lazily load all messages. + * + * @author xiangl@google.com (Xiang Li) + */ +class LazyField { + + final private MessageLite defaultInstance; + final private ExtensionRegistryLite extensionRegistry; + + // Mutable because it is initialized lazily. + private ByteString bytes; + private volatile MessageLite value; + private volatile boolean isDirty = false; + + public LazyField(MessageLite defaultInstance, + ExtensionRegistryLite extensionRegistry, ByteString bytes) { + this.defaultInstance = defaultInstance; + this.extensionRegistry = extensionRegistry; + this.bytes = bytes; + } + + public MessageLite getValue() { + ensureInitialized(); + return value; + } + + /** + * LazyField is not thread-safe for write access. Synchronizations are needed + * under read/write situations. + */ + public MessageLite setValue(MessageLite value) { + MessageLite originalValue = this.value; + this.value = value; + bytes = null; + isDirty = true; + return originalValue; + } + + /** + * Due to the optional field can be duplicated at the end of serialized + * bytes, which will make the serialized size changed after LazyField + * parsed. Be careful when using this method. + */ + public int getSerializedSize() { + if (isDirty) { + return value.getSerializedSize(); + } + return bytes.size(); + } + + public ByteString toByteString() { + if (!isDirty) { + return bytes; + } + synchronized (this) { + if (!isDirty) { + return bytes; + } + bytes = value.toByteString(); + isDirty = false; + return bytes; + } + } + + @Override + public int hashCode() { + ensureInitialized(); + return value.hashCode(); + } + + @Override + public boolean equals(Object obj) { + ensureInitialized(); + return value.equals(obj); + } + + @Override + public String toString() { + ensureInitialized(); + return value.toString(); + } + + private void ensureInitialized() { + if (value != null) { + return; + } + synchronized (this) { + if (value != null) { + return; + } + try { + if (bytes != null) { + value = defaultInstance.getParserForType() + .parseFrom(bytes, extensionRegistry); + } + } catch (IOException e) { + // TODO(xiangl): Refactory the API to support the exception thrown from + // lazily load messages. + } + } + } + + // ==================================================== + + /** + * LazyEntry and LazyIterator are used to encapsulate the LazyField, when + * users iterate all fields from FieldSet. + */ + static class LazyEntry implements Entry { + private Entry entry; + + private LazyEntry(Entry entry) { + this.entry = entry; + } + + @Override + public K getKey() { + return entry.getKey(); + } + + @Override + public Object getValue() { + LazyField field = entry.getValue(); + if (field == null) { + return null; + } + return field.getValue(); + } + + public LazyField getField() { + return entry.getValue(); + } + + @Override + public Object setValue(Object value) { + if (!(value instanceof MessageLite)) { + throw new IllegalArgumentException( + "LazyField now only used for MessageSet, " + + "and the value of MessageSet must be an instance of MessageLite"); + } + return entry.getValue().setValue((MessageLite) value); + } + } + + static class LazyIterator implements Iterator> { + private Iterator> iterator; + + public LazyIterator(Iterator> iterator) { + this.iterator = iterator; + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @SuppressWarnings("unchecked") + @Override + public Entry next() { + Entry entry = iterator.next(); + if (entry.getValue() instanceof LazyField) { + return new LazyEntry((Entry) entry); + } + return (Entry) entry; + } + + @Override + public void remove() { + iterator.remove(); + } + } +} diff --git a/java/src/main/java/com/google/protobuf/LazyStringArrayList.java b/java/src/main/java/com/google/protobuf/LazyStringArrayList.java index 1683a640..75c6a4b7 100644 --- a/java/src/main/java/com/google/protobuf/LazyStringArrayList.java +++ b/java/src/main/java/com/google/protobuf/LazyStringArrayList.java @@ -33,8 +33,9 @@ package com.google.protobuf; import java.util.List; import java.util.AbstractList; import java.util.ArrayList; -import java.util.RandomAccess; import java.util.Collection; +import java.util.Collections; +import java.util.RandomAccess; /** * An implementation of {@link LazyStringList} that wraps an ArrayList. Each @@ -72,6 +73,11 @@ public class LazyStringArrayList extends AbstractList list = new ArrayList(); } + public LazyStringArrayList(LazyStringList from) { + list = new ArrayList(from.size()); + addAll(from); + } + public LazyStringArrayList(List from) { list = new ArrayList(from); } @@ -84,7 +90,7 @@ public class LazyStringArrayList extends AbstractList } else { ByteString bs = (ByteString) o; String s = bs.toStringUtf8(); - if (Internal.isValidUtf8(bs)) { + if (bs.isValidUtf8()) { list.set(index, s); } return s; @@ -108,9 +114,22 @@ public class LazyStringArrayList extends AbstractList modCount++; } + @Override + public boolean addAll(Collection c) { + // The default implementation of AbstractCollection.addAll(Collection) + // delegates to add(Object). This implementation instead delegates to + // addAll(int, Collection), which makes a special case for Collections + // which are instances of LazyStringList. + return addAll(size(), c); + } + @Override public boolean addAll(int index, Collection c) { - boolean ret = list.addAll(index, c); + // When copying from another LazyStringList, directly copy the underlying + // elements rather than forcing each element to be decoded to a String. + Collection collection = c instanceof LazyStringList + ? ((LazyStringList) c).getUnderlyingElements() : c; + boolean ret = list.addAll(index, collection); modCount++; return ret; } @@ -152,4 +171,9 @@ public class LazyStringArrayList extends AbstractList return ((ByteString) o).toStringUtf8(); } } + + @Override + public List getUnderlyingElements() { + return Collections.unmodifiableList(list); + } } diff --git a/java/src/main/java/com/google/protobuf/LazyStringList.java b/java/src/main/java/com/google/protobuf/LazyStringList.java index 97139ca6..630932fe 100644 --- a/java/src/main/java/com/google/protobuf/LazyStringList.java +++ b/java/src/main/java/com/google/protobuf/LazyStringList.java @@ -33,7 +33,7 @@ package com.google.protobuf; import java.util.List; /** - * An interface extending List<String> that also provides access to the + * An interface extending {@code List} that also provides access to the * items of the list as UTF8-encoded ByteString objects. This is used by the * protocol buffer implementation to support lazily converting bytes parsed * over the wire to String objects until needed and also increases the @@ -41,9 +41,9 @@ import java.util.List; * ByteString is already cached. *

* This only adds additional methods that are required for the use in the - * protocol buffer code in order to be able successfuly round trip byte arrays + * protocol buffer code in order to be able successfully round trip byte arrays * through parsing and serialization without conversion to strings. It's not - * attempting to support the functionality of say List<ByteString>, hence + * attempting to support the functionality of say {@code List}, hence * why only these two very specific methods are added. * * @author jonp@google.com (Jon Perlow) @@ -56,7 +56,7 @@ public interface LazyStringList extends List { * @param index index of the element to return * @return the element at the specified position in this list * @throws IndexOutOfBoundsException if the index is out of range - * (index < 0 || index >= size()) + * ({@code index < 0 || index >= size()}) */ ByteString getByteString(int index); @@ -69,4 +69,13 @@ public interface LazyStringList extends List { * is not supported by this list */ void add(ByteString element); + + /** + * Returns an unmodifiable List of the underlying elements, each of + * which is either a {@code String} or its equivalent UTF-8 encoded + * {@code ByteString}. It is an error for the caller to modify the returned + * List, and attempting to do so will result in an + * {@link UnsupportedOperationException}. + */ + List getUnderlyingElements(); } diff --git a/java/src/main/java/com/google/protobuf/LiteralByteString.java b/java/src/main/java/com/google/protobuf/LiteralByteString.java new file mode 100644 index 00000000..93c53dce --- /dev/null +++ b/java/src/main/java/com/google/protobuf/LiteralByteString.java @@ -0,0 +1,349 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// http://code.google.com/p/protobuf/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.NoSuchElementException; + +/** + * This class implements a {@link com.google.protobuf.ByteString} backed by a + * single array of bytes, contiguous in memory. It supports substring by + * pointing to only a sub-range of the underlying byte array, meaning that a + * substring will reference the full byte-array of the string it's made from, + * exactly as with {@link String}. + * + * @author carlanton@google.com (Carl Haverl) + */ +class LiteralByteString extends ByteString { + + protected final byte[] bytes; + + /** + * Creates a {@code LiteralByteString} backed by the given array, without + * copying. + * + * @param bytes array to wrap + */ + LiteralByteString(byte[] bytes) { + this.bytes = bytes; + } + + @Override + public byte byteAt(int index) { + // Unlike most methods in this class, this one is a direct implementation + // ignoring the potential offset because we need to do range-checking in the + // substring case anyway. + return bytes[index]; + } + + @Override + public int size() { + return bytes.length; + } + + // ================================================================= + // ByteString -> substring + + @Override + public ByteString substring(int beginIndex, int endIndex) { + if (beginIndex < 0) { + throw new IndexOutOfBoundsException( + "Beginning index: " + beginIndex + " < 0"); + } + if (endIndex > size()) { + throw new IndexOutOfBoundsException("End index: " + endIndex + " > " + + size()); + } + int substringLength = endIndex - beginIndex; + if (substringLength < 0) { + throw new IndexOutOfBoundsException( + "Beginning index larger than ending index: " + beginIndex + ", " + + endIndex); + } + + ByteString result; + if (substringLength == 0) { + result = ByteString.EMPTY; + } else { + result = new BoundedByteString(bytes, getOffsetIntoBytes() + beginIndex, + substringLength); + } + return result; + } + + // ================================================================= + // ByteString -> byte[] + + @Override + protected void copyToInternal(byte[] target, int sourceOffset, + int targetOffset, int numberToCopy) { + // Optimized form, not for subclasses, since we don't call + // getOffsetIntoBytes() or check the 'numberToCopy' parameter. + System.arraycopy(bytes, sourceOffset, target, targetOffset, numberToCopy); + } + + @Override + public void copyTo(ByteBuffer target) { + target.put(bytes, getOffsetIntoBytes(), size()); // Copies bytes + } + + @Override + public ByteBuffer asReadOnlyByteBuffer() { + ByteBuffer byteBuffer = + ByteBuffer.wrap(bytes, getOffsetIntoBytes(), size()); + return byteBuffer.asReadOnlyBuffer(); + } + + @Override + public List asReadOnlyByteBufferList() { + // Return the ByteBuffer generated by asReadOnlyByteBuffer() as a singleton + List result = new ArrayList(1); + result.add(asReadOnlyByteBuffer()); + return result; + } + + @Override + public void writeTo(OutputStream outputStream) throws IOException { + outputStream.write(toByteArray()); + } + + @Override + public String toString(String charsetName) + throws UnsupportedEncodingException { + return new String(bytes, getOffsetIntoBytes(), size(), charsetName); + } + + // ================================================================= + // UTF-8 decoding + + @Override + public boolean isValidUtf8() { + int offset = getOffsetIntoBytes(); + return Utf8.isValidUtf8(bytes, offset, offset + size()); + } + + @Override + protected int partialIsValidUtf8(int state, int offset, int length) { + int index = getOffsetIntoBytes() + offset; + return Utf8.partialIsValidUtf8(state, bytes, index, index + length); + } + + // ================================================================= + // equals() and hashCode() + + @Override + public boolean equals(Object other) { + if (other == this) { + return true; + } + if (!(other instanceof ByteString)) { + return false; + } + + if (size() != ((ByteString) other).size()) { + return false; + } + if (size() == 0) { + return true; + } + + if (other instanceof LiteralByteString) { + return equalsRange((LiteralByteString) other, 0, size()); + } else if (other instanceof RopeByteString) { + return other.equals(this); + } else { + throw new IllegalArgumentException( + "Has a new type of ByteString been created? Found " + + other.getClass()); + } + } + + /** + * Check equality of the substring of given length of this object starting at + * zero with another {@code LiteralByteString} substring starting at offset. + * + * @param other what to compare a substring in + * @param offset offset into other + * @param length number of bytes to compare + * @return true for equality of substrings, else false. + */ + boolean equalsRange(LiteralByteString other, int offset, int length) { + if (length > other.size()) { + throw new IllegalArgumentException( + "Length too large: " + length + size()); + } + if (offset + length > other.size()) { + throw new IllegalArgumentException( + "Ran off end of other: " + offset + ", " + length + ", " + + other.size()); + } + + byte[] thisBytes = bytes; + byte[] otherBytes = other.bytes; + int thisLimit = getOffsetIntoBytes() + length; + for (int thisIndex = getOffsetIntoBytes(), otherIndex = + other.getOffsetIntoBytes() + offset; + (thisIndex < thisLimit); ++thisIndex, ++otherIndex) { + if (thisBytes[thisIndex] != otherBytes[otherIndex]) { + return false; + } + } + return true; + } + + /** + * Cached hash value. Intentionally accessed via a data race, which + * is safe because of the Java Memory Model's "no out-of-thin-air values" + * guarantees for ints. + */ + private int hash = 0; + + /** + * Compute the hashCode using the traditional algorithm from {@link + * ByteString}. + * + * @return hashCode value + */ + @Override + public int hashCode() { + int h = hash; + + if (h == 0) { + int size = size(); + h = partialHash(size, 0, size); + if (h == 0) { + h = 1; + } + hash = h; + } + return h; + } + + @Override + protected int peekCachedHashCode() { + return hash; + } + + @Override + protected int partialHash(int h, int offset, int length) { + byte[] thisBytes = bytes; + for (int i = getOffsetIntoBytes() + offset, limit = i + length; i < limit; + i++) { + h = h * 31 + thisBytes[i]; + } + return h; + } + + // ================================================================= + // Input stream + + @Override + public InputStream newInput() { + return new ByteArrayInputStream(bytes, getOffsetIntoBytes(), + size()); // No copy + } + + @Override + public CodedInputStream newCodedInput() { + // We trust CodedInputStream not to modify the bytes, or to give anyone + // else access to them. + return CodedInputStream + .newInstance(bytes, getOffsetIntoBytes(), size()); // No copy + } + + // ================================================================= + // ByteIterator + + @Override + public ByteIterator iterator() { + return new LiteralByteIterator(); + } + + private class LiteralByteIterator implements ByteIterator { + private int position; + private final int limit; + + private LiteralByteIterator() { + position = 0; + limit = size(); + } + + public boolean hasNext() { + return (position < limit); + } + + public Byte next() { + // Boxing calls Byte.valueOf(byte), which does not instantiate. + return nextByte(); + } + + public byte nextByte() { + try { + return bytes[position++]; + } catch (ArrayIndexOutOfBoundsException e) { + throw new NoSuchElementException(e.getMessage()); + } + } + + public void remove() { + throw new UnsupportedOperationException(); + } + } + + // ================================================================= + // Internal methods + + @Override + protected int getTreeDepth() { + return 0; + } + + @Override + protected boolean isBalanced() { + return true; + } + + /** + * Offset into {@code bytes[]} to use, non-zero for substrings. + * + * @return always 0 for this class + */ + protected int getOffsetIntoBytes() { + return 0; + } +} diff --git a/java/src/main/java/com/google/protobuf/Message.java b/java/src/main/java/com/google/protobuf/Message.java index 67c4148e..2b881413 100644 --- a/java/src/main/java/com/google/protobuf/Message.java +++ b/java/src/main/java/com/google/protobuf/Message.java @@ -50,25 +50,28 @@ import java.util.Map; */ public interface Message extends MessageLite, MessageOrBuilder { + // (From MessageLite, re-declared here only for return type covariance.) + Parser getParserForType(); + // ----------------------------------------------------------------- // Comparison and hashing /** * Compares the specified object with this message for equality. Returns - * true if the given object is a message of the same type (as + * {@code true} if the given object is a message of the same type (as * defined by {@code getDescriptorForType()}) and has identical values for * all of its fields. Subclasses must implement this; inheriting * {@code Object.equals()} is incorrect. * * @param other object to be compared for equality with this message - * @return true if the specified object is equal to this message + * @return {@code true} if the specified object is equal to this message */ @Override boolean equals(Object other); /** * Returns the hash code value for this message. The hash code of a message - * should mix the message's type (object identity of the decsriptor) with its + * should mix the message's type (object identity of the descriptor) with its * contents (known and unknown field values). Subclasses must implement this; * inheriting {@code Object.hashCode()} is incorrect. * @@ -83,7 +86,8 @@ public interface Message extends MessageLite, MessageOrBuilder { /** * Converts the message to a string in protocol buffer text format. This is - * just a trivial wrapper around {@link TextFormat#printToString(Message)}. + * just a trivial wrapper around {@link + * TextFormat#printToString(MessageOrBuilder)}. */ @Override String toString(); @@ -144,6 +148,24 @@ public interface Message extends MessageLite, MessageOrBuilder { */ Builder newBuilderForField(Descriptors.FieldDescriptor field); + /** + * Get a nested builder instance for the given field. + *

+ * Normally, we hold a reference to the immutable message object for the + * message type field. Some implementations(the generated message builders), + * however, can also hold a reference to the builder object (a nested + * builder) for the field. + *

+ * If the field is already backed up by a nested builder, the nested builder + * will be returned. Otherwise, a new field builder will be created and + * returned. The original message field (if exist) will be merged into the + * field builder, which will then be nested into its parent builder. + *

+ * NOTE: implementations that do not support nested builders will throw + * UnsupportedException. + */ + Builder getFieldBuilder(Descriptors.FieldDescriptor field); + /** * Sets a field to the given value. The value must be of the correct type * for this field, i.e. the same type that diff --git a/java/src/main/java/com/google/protobuf/MessageLite.java b/java/src/main/java/com/google/protobuf/MessageLite.java index 31b8256e..e5b9a47b 100644 --- a/java/src/main/java/com/google/protobuf/MessageLite.java +++ b/java/src/main/java/com/google/protobuf/MessageLite.java @@ -79,6 +79,12 @@ public interface MessageLite extends MessageLiteOrBuilder { */ int getSerializedSize(); + + /** + * Gets the parser for a message of the same type as this message. + */ + Parser getParserForType(); + // ----------------------------------------------------------------- // Convenience methods. @@ -144,11 +150,8 @@ public interface MessageLite extends MessageLiteOrBuilder { Builder clear(); /** - * Construct the final message. Once this is called, the Builder is no - * longer valid, and calling any other method will result in undefined - * behavior and may throw a NullPointerException. If you need to continue - * working with the builder after calling {@code build()}, {@code clone()} - * it first. + * Constructs the message based on the state of the Builder. Subsequent + * changes to the Builder will not affect the returned message. * @throws UninitializedMessageException The message is missing one or more * required fields (i.e. {@link #isInitialized()} returns false). * Use {@link #buildPartial()} to bypass this check. @@ -158,11 +161,7 @@ public interface MessageLite extends MessageLiteOrBuilder { /** * Like {@link #build()}, but does not throw an exception if the message * is missing required fields. Instead, a partial message is returned. - * Once this is called, the Builder is no longer valid, and calling any - * will result in undefined behavior and may throw a NullPointerException. - * - * If you need to continue working with the builder after calling - * {@code buildPartial()}, {@code clone()} it first. + * Subsequent changes to the Builder will not affect the returned message. */ MessageLite buildPartial(); @@ -174,7 +173,7 @@ public interface MessageLite extends MessageLiteOrBuilder { /** * Parses a message of this type from the input and merges it with this - * message, as if using {@link Builder#mergeFrom(MessageLite)}. + * message. * *

Warning: This does not verify that all required fields are present in * the input message. If you call {@link #build()} without setting all @@ -184,11 +183,6 @@ public interface MessageLite extends MessageLiteOrBuilder { *

    *
  • Call {@link #isInitialized()} to verify that all required fields * are set before building. - *
  • Parse the message separately using one of the static - * {@code parseFrom} methods, then use {@link #mergeFrom(MessageLite)} - * to merge it with this one. {@code parseFrom} will throw an - * {@link InvalidProtocolBufferException} (an {@code IOException}) - * if some required fields are missing. *
  • Use {@code buildPartial()} to build, which ignores missing * required fields. *
@@ -225,7 +219,7 @@ public interface MessageLite extends MessageLiteOrBuilder { /** * Parse {@code data} as a message of this type and merge it with the * message being built. This is just a small wrapper around - * {@link #mergeFrom(CodedInputStream,ExtensionRegistry)}. + * {@link #mergeFrom(CodedInputStream,ExtensionRegistryLite)}. * * @return this */ @@ -255,7 +249,7 @@ public interface MessageLite extends MessageLiteOrBuilder { /** * Parse {@code data} as a message of this type and merge it with the * message being built. This is just a small wrapper around - * {@link #mergeFrom(CodedInputStream,ExtensionRegistry)}. + * {@link #mergeFrom(CodedInputStream,ExtensionRegistryLite)}. * * @return this */ @@ -266,7 +260,7 @@ public interface MessageLite extends MessageLiteOrBuilder { /** * Parse {@code data} as a message of this type and merge it with the * message being built. This is just a small wrapper around - * {@link #mergeFrom(CodedInputStream,ExtensionRegistry)}. + * {@link #mergeFrom(CodedInputStream,ExtensionRegistryLite)}. * * @return this */ @@ -293,7 +287,7 @@ public interface MessageLite extends MessageLiteOrBuilder { /** * Parse a message of this type from {@code input} and merge it with the * message being built. This is just a small wrapper around - * {@link #mergeFrom(CodedInputStream,ExtensionRegistry)}. + * {@link #mergeFrom(CodedInputStream,ExtensionRegistryLite)}. * * @return this */ @@ -308,9 +302,9 @@ public interface MessageLite extends MessageLiteOrBuilder { * {@link MessageLite#writeDelimitedTo(OutputStream)} to write messages in * this format. * - * @returns True if successful, or false if the stream is at EOF when the - * method starts. Any other error (including reaching EOF during - * parsing) will cause an exception to be thrown. + * @return True if successful, or false if the stream is at EOF when the + * method starts. Any other error (including reaching EOF during + * parsing) will cause an exception to be thrown. */ boolean mergeDelimitedFrom(InputStream input) throws IOException; diff --git a/java/src/main/java/com/google/protobuf/MessageLiteOrBuilder.java b/java/src/main/java/com/google/protobuf/MessageLiteOrBuilder.java index 7cc72e9c..05b2b161 100644 --- a/java/src/main/java/com/google/protobuf/MessageLiteOrBuilder.java +++ b/java/src/main/java/com/google/protobuf/MessageLiteOrBuilder.java @@ -52,6 +52,8 @@ public interface MessageLiteOrBuilder { /** * Returns true if all required fields in the message and all embedded * messages are set, false otherwise. + * + *

See also: {@link MessageOrBuilder#getInitializationErrorString()} */ boolean isInitialized(); diff --git a/java/src/main/java/com/google/protobuf/MessageOrBuilder.java b/java/src/main/java/com/google/protobuf/MessageOrBuilder.java index 0132e7ca..bf62d45e 100644 --- a/java/src/main/java/com/google/protobuf/MessageOrBuilder.java +++ b/java/src/main/java/com/google/protobuf/MessageOrBuilder.java @@ -30,6 +30,7 @@ package com.google.protobuf; +import java.util.List; import java.util.Map; /** @@ -44,6 +45,24 @@ public interface MessageOrBuilder extends MessageLiteOrBuilder { //@Override (Java 1.6 override semantics, but we must support 1.5) Message getDefaultInstanceForType(); + /** + * Returns a list of field paths (e.g. "foo.bar.baz") of required fields + * which are not set in this message. You should call + * {@link MessageLiteOrBuilder#isInitialized()} first to check if there + * are any missing fields, as that method is likely to be much faster + * than this one even when the message is fully-initialized. + */ + List findInitializationErrors(); + + /** + * Returns a comma-delimited list of required fields which are not set + * in this message object. You should call + * {@link MessageLiteOrBuilder#isInitialized()} first to check if there + * are any missing fields, as that method is likely to be much faster + * than this one even when the message is fully-initialized. + */ + String getInitializationErrorString(); + /** * Get the message's type's descriptor. This differs from the * {@code getDescriptor()} method of generated message classes in that @@ -80,7 +99,7 @@ public interface MessageOrBuilder extends MessageLiteOrBuilder { /** * Obtains the value of the given field, or the default value if it is * not set. For primitive fields, the boxed primitive value is returned. - * For enum fields, the EnumValueDescriptor for the value is returend. For + * For enum fields, the EnumValueDescriptor for the value is returned. For * embedded message fields, the sub-message is returned. For repeated * fields, a java.util.List is returned. */ @@ -98,7 +117,7 @@ public interface MessageOrBuilder extends MessageLiteOrBuilder { /** * Gets an element of a repeated field. For primitive fields, the boxed * primitive value is returned. For enum fields, the EnumValueDescriptor - * for the value is returend. For embedded message fields, the sub-message + * for the value is returned. For embedded message fields, the sub-message * is returned. * @throws IllegalArgumentException The field is not a repeated field, or * {@code field.getContainingType() != getDescriptorForType()}. diff --git a/java/src/main/java/com/google/protobuf/Parser.java b/java/src/main/java/com/google/protobuf/Parser.java new file mode 100644 index 00000000..7d8e8217 --- /dev/null +++ b/java/src/main/java/com/google/protobuf/Parser.java @@ -0,0 +1,259 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// http://code.google.com/p/protobuf/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import java.io.InputStream; + +/** + * Abstract interface for parsing Protocol Messages. + * + * @author liujisi@google.com (Pherl Liu) + */ +public interface Parser { + /** + * Parses a message of {@code MessageType} from the input. + * + *

Note: The caller should call + * {@link CodedInputStream#checkLastTagWas(int)} after calling this to + * verify that the last tag seen was the appropriate end-group tag, + * or zero for EOF. + */ + public MessageType parseFrom(CodedInputStream input) + throws InvalidProtocolBufferException; + + /** + * Like {@link #parseFrom(CodedInputStream)}, but also parses extensions. + * The extensions that you want to be able to parse must be registered in + * {@code extensionRegistry}. Extensions not in the registry will be treated + * as unknown fields. + */ + public MessageType parseFrom(CodedInputStream input, + ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException; + + /** + * Like {@link #parseFrom(CodedInputStream)}, but does not throw an + * exception if the message is missing required fields. Instead, a partial + * message is returned. + */ + public MessageType parsePartialFrom(CodedInputStream input) + throws InvalidProtocolBufferException; + + /** + * Like {@link #parseFrom(CodedInputStream input, ExtensionRegistryLite)}, + * but does not throw an exception if the message is missing required fields. + * Instead, a partial message is returned. + */ + public MessageType parsePartialFrom(CodedInputStream input, + ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException; + + // --------------------------------------------------------------- + // Convenience methods. + + /** + * Parses {@code data} as a message of {@code MessageType}. + * This is just a small wrapper around {@link #parseFrom(CodedInputStream)}. + */ + public MessageType parseFrom(ByteString data) + throws InvalidProtocolBufferException; + + /** + * Parses {@code data} as a message of {@code MessageType}. + * This is just a small wrapper around + * {@link #parseFrom(CodedInputStream, ExtensionRegistryLite)}. + */ + public MessageType parseFrom(ByteString data, + ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException; + + /** + * Like {@link #parseFrom(ByteString)}, but does not throw an + * exception if the message is missing required fields. Instead, a partial + * message is returned. + */ + public MessageType parsePartialFrom(ByteString data) + throws InvalidProtocolBufferException; + + /** + * Like {@link #parseFrom(ByteString, ExtensionRegistryLite)}, + * but does not throw an exception if the message is missing required fields. + * Instead, a partial message is returned. + */ + public MessageType parsePartialFrom(ByteString data, + ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException; + + /** + * Parses {@code data} as a message of {@code MessageType}. + * This is just a small wrapper around {@link #parseFrom(CodedInputStream)}. + */ + public MessageType parseFrom(byte[] data, int off, int len) + throws InvalidProtocolBufferException; + + /** + * Parses {@code data} as a message of {@code MessageType}. + * This is just a small wrapper around + * {@link #parseFrom(CodedInputStream, ExtensionRegistryLite)}. + */ + public MessageType parseFrom(byte[] data, int off, int len, + ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException; + + /** + * Parses {@code data} as a message of {@code MessageType}. + * This is just a small wrapper around {@link #parseFrom(CodedInputStream)}. + */ + public MessageType parseFrom(byte[] data) + throws InvalidProtocolBufferException; + + /** + * Parses {@code data} as a message of {@code MessageType}. + * This is just a small wrapper around + * {@link #parseFrom(CodedInputStream, ExtensionRegistryLite)}. + */ + public MessageType parseFrom(byte[] data, + ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException; + + /** + * Like {@link #parseFrom(byte[], int, int)}, but does not throw an + * exception if the message is missing required fields. Instead, a partial + * message is returned. + */ + public MessageType parsePartialFrom(byte[] data, int off, int len) + throws InvalidProtocolBufferException; + + /** + * Like {@link #parseFrom(ByteString, ExtensionRegistryLite)}, + * but does not throw an exception if the message is missing required fields. + * Instead, a partial message is returned. + */ + public MessageType parsePartialFrom(byte[] data, int off, int len, + ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException; + + /** + * Like {@link #parseFrom(byte[])}, but does not throw an + * exception if the message is missing required fields. Instead, a partial + * message is returned. + */ + public MessageType parsePartialFrom(byte[] data) + throws InvalidProtocolBufferException; + + /** + * Like {@link #parseFrom(byte[], ExtensionRegistryLite)}, + * but does not throw an exception if the message is missing required fields. + * Instead, a partial message is returned. + */ + public MessageType parsePartialFrom(byte[] data, + ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException; + + /** + * Parse a message of {@code MessageType} from {@code input}. + * This is just a small wrapper around {@link #parseFrom(CodedInputStream)}. + * Note that this method always reads the entire input (unless it + * throws an exception). If you want it to stop earlier, you will need to + * wrap your input in some wrapper stream that limits reading. Or, use + * {@link MessageLite#writeDelimitedTo(java.io.OutputStream)} to write your + * message and {@link #parseDelimitedFrom(InputStream)} to read it. + *

+ * Despite usually reading the entire input, this does not close the stream. + */ + public MessageType parseFrom(InputStream input) + throws InvalidProtocolBufferException; + + /** + * Parses a message of {@code MessageType} from {@code input}. + * This is just a small wrapper around + * {@link #parseFrom(CodedInputStream, ExtensionRegistryLite)}. + */ + public MessageType parseFrom(InputStream input, + ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException; + + /** + * Like {@link #parseFrom(InputStream)}, but does not throw an + * exception if the message is missing required fields. Instead, a partial + * message is returned. + */ + public MessageType parsePartialFrom(InputStream input) + throws InvalidProtocolBufferException; + + /** + * Like {@link #parseFrom(InputStream, ExtensionRegistryLite)}, + * but does not throw an exception if the message is missing required fields. + * Instead, a partial message is returned. + */ + public MessageType parsePartialFrom(InputStream input, + ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException; + + /** + * Like {@link #parseFrom(InputStream)}, but does not read util EOF. + * Instead, the size of message (encoded as a varint) is read first, + * then the message data. Use + * {@link MessageLite#writeDelimitedTo(java.io.OutputStream)} to write + * messages in this format. + * + * @return True if successful, or false if the stream is at EOF when the + * method starts. Any other error (including reaching EOF during + * parsing) will cause an exception to be thrown. + */ + public MessageType parseDelimitedFrom(InputStream input) + throws InvalidProtocolBufferException; + + /** + * Like {@link #parseDelimitedFrom(InputStream)} but supporting extensions. + */ + public MessageType parseDelimitedFrom(InputStream input, + ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException; + + /** + * Like {@link #parseDelimitedFrom(InputStream)}, but does not throw an + * exception if the message is missing required fields. Instead, a partial + * message is returned. + */ + public MessageType parsePartialDelimitedFrom(InputStream input) + throws InvalidProtocolBufferException; + + /** + * Like {@link #parseDelimitedFrom(InputStream, ExtensionRegistryLite)}, + * but does not throw an exception if the message is missing required fields. + * Instead, a partial message is returned. + */ + public MessageType parsePartialDelimitedFrom( + InputStream input, + ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException; +} diff --git a/java/src/main/java/com/google/protobuf/RepeatedFieldBuilder.java b/java/src/main/java/com/google/protobuf/RepeatedFieldBuilder.java index 0024f791..65d9270d 100644 --- a/java/src/main/java/com/google/protobuf/RepeatedFieldBuilder.java +++ b/java/src/main/java/com/google/protobuf/RepeatedFieldBuilder.java @@ -37,22 +37,22 @@ import java.util.Collections; import java.util.List; /** - * RepeatedFieldBuilder implements a structure that a protocol + * {@code RepeatedFieldBuilder} implements a structure that a protocol * message uses to hold a repeated field of other protocol messages. It supports * the classical use case of adding immutable {@link Message}'s to the * repeated field and is highly optimized around this (no extra memory * allocations and sharing of immutable arrays). *
* It also supports the additional use case of adding a {@link Message.Builder} - * to the repeated field and deferring conversion of that Builder - * to an immutable Message. In this way, it's possible to maintain - * a tree of Builder's that acts as a fully read/write data + * to the repeated field and deferring conversion of that {@code Builder} + * to an immutable {@code Message}. In this way, it's possible to maintain + * a tree of {@code Builder}'s that acts as a fully read/write data * structure. *
* Logically, one can think of a tree of builders as converting the entire tree * to messages when build is called on the root or when any method is called * that desires a Message instead of a Builder. In terms of the implementation, - * the SingleFieldBuilder and RepeatedFieldBuilder + * the {@code SingleFieldBuilder} and {@code RepeatedFieldBuilder} * classes cache messages that were created so that messages only need to be * created when some change occured in its builder or a builder for one of its * descendants. @@ -192,7 +192,7 @@ public class RepeatedFieldBuilder /** * Get the message at the specified index. If the message is currently stored - * as a Builder, it is converted to a Message by + * as a {@code Builder}, it is converted to a {@code Message} by * calling {@link Message.Builder#buildPartial} on it. * * @param index the index of the message to get @@ -204,7 +204,7 @@ public class RepeatedFieldBuilder /** * Get the message at the specified index. If the message is currently stored - * as a Builder, it is converted to a Message by + * as a {@code Builder}, it is converted to a {@code Message} by * calling {@link Message.Builder#buildPartial} on it. * * @param index the index of the message to get diff --git a/java/src/main/java/com/google/protobuf/RopeByteString.java b/java/src/main/java/com/google/protobuf/RopeByteString.java new file mode 100644 index 00000000..8d44d117 --- /dev/null +++ b/java/src/main/java/com/google/protobuf/RopeByteString.java @@ -0,0 +1,945 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// http://code.google.com/p/protobuf/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.io.ByteArrayInputStream; +import java.nio.ByteBuffer; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Deque; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; + +/** + * Class to represent {@code ByteStrings} formed by concatenation of other + * ByteStrings, without copying the data in the pieces. The concatenation is + * represented as a tree whose leaf nodes are each a {@link LiteralByteString}. + * + *

Most of the operation here is inspired by the now-famous paper + * BAP95 Ropes: an Alternative to Strings hans-j. boehm, russ atkinson and + * michael plass + * + *

The algorithms described in the paper have been implemented for character + * strings in {@link com.google.common.string.Rope} and in the c++ class {@code + * cord.cc}. + * + *

Fundamentally the Rope algorithm represents the collection of pieces as a + * binary tree. BAP95 uses a Fibonacci bound relating depth to a minimum + * sequence length, sequences that are too short relative to their depth cause a + * tree rebalance. More precisely, a tree of depth d is "balanced" in the + * terminology of BAP95 if its length is at least F(d+2), where F(n) is the + * n-the Fibonacci number. Thus for depths 0, 1, 2, 3, 4, 5,... we have minimum + * lengths 1, 2, 3, 5, 8, 13,... + * + * @author carlanton@google.com (Carl Haverl) + */ +class RopeByteString extends ByteString { + + /** + * BAP95. Let Fn be the nth Fibonacci number. A {@link RopeByteString} of + * depth n is "balanced", i.e flat enough, if its length is at least Fn+2, + * e.g. a "balanced" {@link RopeByteString} of depth 1 must have length at + * least 2, of depth 4 must have length >= 8, etc. + * + *

There's nothing special about using the Fibonacci numbers for this, but + * they are a reasonable sequence for encapsulating the idea that we are OK + * with longer strings being encoded in deeper binary trees. + * + *

For 32-bit integers, this array has length 46. + */ + private static final int[] minLengthByDepth; + + static { + // Dynamically generate the list of Fibonacci numbers the first time this + // class is accessed. + List numbers = new ArrayList(); + + // we skip the first Fibonacci number (1). So instead of: 1 1 2 3 5 8 ... + // we have: 1 2 3 5 8 ... + int f1 = 1; + int f2 = 1; + + // get all the values until we roll over. + while (f2 > 0) { + numbers.add(f2); + int temp = f1 + f2; + f1 = f2; + f2 = temp; + } + + // we include this here so that we can index this array to [x + 1] in the + // loops below. + numbers.add(Integer.MAX_VALUE); + minLengthByDepth = new int[numbers.size()]; + for (int i = 0; i < minLengthByDepth.length; i++) { + // unbox all the values + minLengthByDepth[i] = numbers.get(i); + } + } + + private final int totalLength; + private final ByteString left; + private final ByteString right; + private final int leftLength; + private final int treeDepth; + + /** + * Create a new RopeByteString, which can be thought of as a new tree node, by + * recording references to the two given strings. + * + * @param left string on the left of this node, should have {@code size() > + * 0} + * @param right string on the right of this node, should have {@code size() > + * 0} + */ + private RopeByteString(ByteString left, ByteString right) { + this.left = left; + this.right = right; + leftLength = left.size(); + totalLength = leftLength + right.size(); + treeDepth = Math.max(left.getTreeDepth(), right.getTreeDepth()) + 1; + } + + /** + * Concatenate the given strings while performing various optimizations to + * slow the growth rate of tree depth and tree node count. The result is + * either a {@link LiteralByteString} or a {@link RopeByteString} + * depending on which optimizations, if any, were applied. + * + *

Small pieces of length less than {@link + * ByteString#CONCATENATE_BY_COPY_SIZE} may be copied by value here, as in + * BAP95. Large pieces are referenced without copy. + * + * @param left string on the left + * @param right string on the right + * @return concatenation representing the same sequence as the given strings + */ + static ByteString concatenate(ByteString left, ByteString right) { + ByteString result; + RopeByteString leftRope = + (left instanceof RopeByteString) ? (RopeByteString) left : null; + if (right.size() == 0) { + result = left; + } else if (left.size() == 0) { + result = right; + } else { + int newLength = left.size() + right.size(); + if (newLength < ByteString.CONCATENATE_BY_COPY_SIZE) { + // Optimization from BAP95: For short (leaves in paper, but just short + // here) total length, do a copy of data to a new leaf. + result = concatenateBytes(left, right); + } else if (leftRope != null + && leftRope.right.size() + right.size() < CONCATENATE_BY_COPY_SIZE) { + // Optimization from BAP95: As an optimization of the case where the + // ByteString is constructed by repeated concatenate, recognize the case + // where a short string is concatenated to a left-hand node whose + // right-hand branch is short. In the paper this applies to leaves, but + // we just look at the length here. This has the advantage of shedding + // references to unneeded data when substrings have been taken. + // + // When we recognize this case, we do a copy of the data and create a + // new parent node so that the depth of the result is the same as the + // given left tree. + ByteString newRight = concatenateBytes(leftRope.right, right); + result = new RopeByteString(leftRope.left, newRight); + } else if (leftRope != null + && leftRope.left.getTreeDepth() > leftRope.right.getTreeDepth() + && leftRope.getTreeDepth() > right.getTreeDepth()) { + // Typically for concatenate-built strings the left-side is deeper than + // the right. This is our final attempt to concatenate without + // increasing the tree depth. We'll redo the the node on the RHS. This + // is yet another optimization for building the string by repeatedly + // concatenating on the right. + ByteString newRight = new RopeByteString(leftRope.right, right); + result = new RopeByteString(leftRope.left, newRight); + } else { + // Fine, we'll add a node and increase the tree depth--unless we + // rebalance ;^) + int newDepth = Math.max(left.getTreeDepth(), right.getTreeDepth()) + 1; + if (newLength >= minLengthByDepth[newDepth]) { + // The tree is shallow enough, so don't rebalance + result = new RopeByteString(left, right); + } else { + result = new Balancer().balance(left, right); + } + } + } + return result; + } + + /** + * Concatenates two strings by copying data values. This is called in a few + * cases in order to reduce the growth of the number of tree nodes. + * + * @param left string on the left + * @param right string on the right + * @return string formed by copying data bytes + */ + private static LiteralByteString concatenateBytes(ByteString left, + ByteString right) { + int leftSize = left.size(); + int rightSize = right.size(); + byte[] bytes = new byte[leftSize + rightSize]; + left.copyTo(bytes, 0, 0, leftSize); + right.copyTo(bytes, 0, leftSize, rightSize); + return new LiteralByteString(bytes); // Constructor wraps bytes + } + + /** + * Create a new RopeByteString for testing only while bypassing all the + * defenses of {@link #concatenate(ByteString, ByteString)}. This allows + * testing trees of specific structure. We are also able to insert empty + * leaves, though these are dis-allowed, so that we can make sure the + * implementation can withstand their presence. + * + * @param left string on the left of this node + * @param right string on the right of this node + * @return an unsafe instance for testing only + */ + static RopeByteString newInstanceForTest(ByteString left, ByteString right) { + return new RopeByteString(left, right); + } + + /** + * Gets the byte at the given index. + * Throws {@link ArrayIndexOutOfBoundsException} for backwards-compatibility + * reasons although it would more properly be {@link + * IndexOutOfBoundsException}. + * + * @param index index of byte + * @return the value + * @throws ArrayIndexOutOfBoundsException {@code index} is < 0 or >= size + */ + @Override + public byte byteAt(int index) { + if (index < 0) { + throw new ArrayIndexOutOfBoundsException("Index < 0: " + index); + } + if (index > totalLength) { + throw new ArrayIndexOutOfBoundsException( + "Index > length: " + index + ", " + totalLength); + } + + byte result; + // Find the relevant piece by recursive descent + if (index < leftLength) { + result = left.byteAt(index); + } else { + result = right.byteAt(index - leftLength); + } + return result; + } + + @Override + public int size() { + return totalLength; + } + + // ================================================================= + // Pieces + + @Override + protected int getTreeDepth() { + return treeDepth; + } + + /** + * Determines if the tree is balanced according to BAP95, which means the tree + * is flat-enough with respect to the bounds. Note that this definition of + * balanced is one where sub-trees of balanced trees are not necessarily + * balanced. + * + * @return true if the tree is balanced + */ + @Override + protected boolean isBalanced() { + return totalLength >= minLengthByDepth[treeDepth]; + } + + /** + * Takes a substring of this one. This involves recursive descent along the + * left and right edges of the substring, and referencing any wholly contained + * segments in between. Any leaf nodes entirely uninvolved in the substring + * will not be referenced by the substring. + * + *

Substrings of {@code length < 2} should result in at most a single + * recursive call chain, terminating at a leaf node. Thus the result will be a + * {@link LiteralByteString}. {@link #RopeByteString(ByteString, + * ByteString)}. + * + * @param beginIndex start at this index + * @param endIndex the last character is the one before this index + * @return substring leaf node or tree + */ + @Override + public ByteString substring(int beginIndex, int endIndex) { + if (beginIndex < 0) { + throw new IndexOutOfBoundsException( + "Beginning index: " + beginIndex + " < 0"); + } + if (endIndex > totalLength) { + throw new IndexOutOfBoundsException( + "End index: " + endIndex + " > " + totalLength); + } + int substringLength = endIndex - beginIndex; + if (substringLength < 0) { + throw new IndexOutOfBoundsException( + "Beginning index larger than ending index: " + beginIndex + ", " + + endIndex); + } + + ByteString result; + if (substringLength == 0) { + // Empty substring + result = ByteString.EMPTY; + } else if (substringLength == totalLength) { + // The whole string + result = this; + } else { + // Proper substring + if (endIndex <= leftLength) { + // Substring on the left + result = left.substring(beginIndex, endIndex); + } else if (beginIndex >= leftLength) { + // Substring on the right + result = right + .substring(beginIndex - leftLength, endIndex - leftLength); + } else { + // Split substring + ByteString leftSub = left.substring(beginIndex); + ByteString rightSub = right.substring(0, endIndex - leftLength); + // Intentionally not rebalancing, since in many cases these two + // substrings will already be less deep than the top-level + // RopeByteString we're taking a substring of. + result = new RopeByteString(leftSub, rightSub); + } + } + return result; + } + + // ================================================================= + // ByteString -> byte[] + + @Override + protected void copyToInternal(byte[] target, int sourceOffset, + int targetOffset, int numberToCopy) { + if (sourceOffset + numberToCopy <= leftLength) { + left.copyToInternal(target, sourceOffset, targetOffset, numberToCopy); + } else if (sourceOffset >= leftLength) { + right.copyToInternal(target, sourceOffset - leftLength, targetOffset, + numberToCopy); + } else { + int leftLength = this.leftLength - sourceOffset; + left.copyToInternal(target, sourceOffset, targetOffset, leftLength); + right.copyToInternal(target, 0, targetOffset + leftLength, + numberToCopy - leftLength); + } + } + + @Override + public void copyTo(ByteBuffer target) { + left.copyTo(target); + right.copyTo(target); + } + + @Override + public ByteBuffer asReadOnlyByteBuffer() { + ByteBuffer byteBuffer = ByteBuffer.wrap(toByteArray()); + return byteBuffer.asReadOnlyBuffer(); + } + + @Override + public List asReadOnlyByteBufferList() { + // Walk through the list of LiteralByteString's that make up this + // rope, and add each one as a read-only ByteBuffer. + List result = new ArrayList(); + PieceIterator pieces = new PieceIterator(this); + while (pieces.hasNext()) { + LiteralByteString byteString = pieces.next(); + result.add(byteString.asReadOnlyByteBuffer()); + } + return result; + } + + @Override + public void writeTo(OutputStream outputStream) throws IOException { + left.writeTo(outputStream); + right.writeTo(outputStream); + } + + @Override + public String toString(String charsetName) + throws UnsupportedEncodingException { + return new String(toByteArray(), charsetName); + } + + // ================================================================= + // UTF-8 decoding + + @Override + public boolean isValidUtf8() { + int leftPartial = left.partialIsValidUtf8(Utf8.COMPLETE, 0, leftLength); + int state = right.partialIsValidUtf8(leftPartial, 0, right.size()); + return state == Utf8.COMPLETE; + } + + @Override + protected int partialIsValidUtf8(int state, int offset, int length) { + int toIndex = offset + length; + if (toIndex <= leftLength) { + return left.partialIsValidUtf8(state, offset, length); + } else if (offset >= leftLength) { + return right.partialIsValidUtf8(state, offset - leftLength, length); + } else { + int leftLength = this.leftLength - offset; + int leftPartial = left.partialIsValidUtf8(state, offset, leftLength); + return right.partialIsValidUtf8(leftPartial, 0, length - leftLength); + } + } + + // ================================================================= + // equals() and hashCode() + + @Override + public boolean equals(Object other) { + if (other == this) { + return true; + } + if (!(other instanceof ByteString)) { + return false; + } + + ByteString otherByteString = (ByteString) other; + if (totalLength != otherByteString.size()) { + return false; + } + if (totalLength == 0) { + return true; + } + + // You don't really want to be calling equals on long strings, but since + // we cache the hashCode, we effectively cache inequality. We use the cached + // hashCode if it's already computed. It's arguable we should compute the + // hashCode here, and if we're going to be testing a bunch of byteStrings, + // it might even make sense. + if (hash != 0) { + int cachedOtherHash = otherByteString.peekCachedHashCode(); + if (cachedOtherHash != 0 && hash != cachedOtherHash) { + return false; + } + } + + return equalsFragments(otherByteString); + } + + /** + * Determines if this string is equal to another of the same length by + * iterating over the leaf nodes. On each step of the iteration, the + * overlapping segments of the leaves are compared. + * + * @param other string of the same length as this one + * @return true if the values of this string equals the value of the given + * one + */ + private boolean equalsFragments(ByteString other) { + int thisOffset = 0; + Iterator thisIter = new PieceIterator(this); + LiteralByteString thisString = thisIter.next(); + + int thatOffset = 0; + Iterator thatIter = new PieceIterator(other); + LiteralByteString thatString = thatIter.next(); + + int pos = 0; + while (true) { + int thisRemaining = thisString.size() - thisOffset; + int thatRemaining = thatString.size() - thatOffset; + int bytesToCompare = Math.min(thisRemaining, thatRemaining); + + // At least one of the offsets will be zero + boolean stillEqual = (thisOffset == 0) + ? thisString.equalsRange(thatString, thatOffset, bytesToCompare) + : thatString.equalsRange(thisString, thisOffset, bytesToCompare); + if (!stillEqual) { + return false; + } + + pos += bytesToCompare; + if (pos >= totalLength) { + if (pos == totalLength) { + return true; + } + throw new IllegalStateException(); + } + // We always get to the end of at least one of the pieces + if (bytesToCompare == thisRemaining) { // If reached end of this + thisOffset = 0; + thisString = thisIter.next(); + } else { + thisOffset += bytesToCompare; + } + if (bytesToCompare == thatRemaining) { // If reached end of that + thatOffset = 0; + thatString = thatIter.next(); + } else { + thatOffset += bytesToCompare; + } + } + } + + /** + * Cached hash value. Intentionally accessed via a data race, which is safe + * because of the Java Memory Model's "no out-of-thin-air values" guarantees + * for ints. + */ + private int hash = 0; + + @Override + public int hashCode() { + int h = hash; + + if (h == 0) { + h = totalLength; + h = partialHash(h, 0, totalLength); + if (h == 0) { + h = 1; + } + hash = h; + } + return h; + } + + @Override + protected int peekCachedHashCode() { + return hash; + } + + @Override + protected int partialHash(int h, int offset, int length) { + int toIndex = offset + length; + if (toIndex <= leftLength) { + return left.partialHash(h, offset, length); + } else if (offset >= leftLength) { + return right.partialHash(h, offset - leftLength, length); + } else { + int leftLength = this.leftLength - offset; + int leftPartial = left.partialHash(h, offset, leftLength); + return right.partialHash(leftPartial, 0, length - leftLength); + } + } + + // ================================================================= + // Input stream + + @Override + public CodedInputStream newCodedInput() { + return CodedInputStream.newInstance(new RopeInputStream()); + } + + @Override + public InputStream newInput() { + return new RopeInputStream(); + } + + /** + * This class implements the balancing algorithm of BAP95. In the paper the + * authors use an array to keep track of pieces, while here we use a stack. + * The tree is balanced by traversing subtrees in left to right order, and the + * stack always contains the part of the string we've traversed so far. + * + *

One surprising aspect of the algorithm is the result of balancing is not + * necessarily balanced, though it is nearly balanced. For details, see + * BAP95. + */ + private static class Balancer { + // Stack containing the part of the string, starting from the left, that + // we've already traversed. The final string should be the equivalent of + // concatenating the strings on the stack from bottom to top. + private final Deque prefixesStack = + new ArrayDeque(minLengthByDepth.length); + + private ByteString balance(ByteString left, ByteString right) { + doBalance(left); + doBalance(right); + + // Sweep stack to gather the result + ByteString partialString = prefixesStack.pop(); + while (!prefixesStack.isEmpty()) { + ByteString newLeft = prefixesStack.pop(); + partialString = new RopeByteString(newLeft, partialString); + } + // We should end up with a RopeByteString since at a minimum we will + // create one from concatenating left and right + return partialString; + } + + private void doBalance(ByteString root) { + // BAP95: Insert balanced subtrees whole. This means the result might not + // be balanced, leading to repeated rebalancings on concatenate. However, + // these rebalancings are shallow due to ignoring balanced subtrees, and + // relatively few calls to insert() result. + if (root.isBalanced()) { + insert(root); + } else if (root instanceof RopeByteString) { + RopeByteString rbs = (RopeByteString) root; + doBalance(rbs.left); + doBalance(rbs.right); + } else { + throw new IllegalArgumentException( + "Has a new type of ByteString been created? Found " + + root.getClass()); + } + } + + /** + * Push a string on the balance stack (BAP95). BAP95 uses an array and + * calls the elements in the array 'bins'. We instead use a stack, so the + * 'bins' of lengths are represented by differences between the elements of + * minLengthByDepth. + * + *

If the length bin for our string, and all shorter length bins, are + * empty, we just push it on the stack. Otherwise, we need to start + * concatenating, putting the given string in the "middle" and continuing + * until we land in an empty length bin that matches the length of our + * concatenation. + * + * @param byteString string to place on the balance stack + */ + private void insert(ByteString byteString) { + int depthBin = getDepthBinForLength(byteString.size()); + int binEnd = minLengthByDepth[depthBin + 1]; + + // BAP95: Concatenate all trees occupying bins representing the length of + // our new piece or of shorter pieces, to the extent that is possible. + // The goal is to clear the bin which our piece belongs in, but that may + // not be entirely possible if there aren't enough longer bins occupied. + if (prefixesStack.isEmpty() || prefixesStack.peek().size() >= binEnd) { + prefixesStack.push(byteString); + } else { + int binStart = minLengthByDepth[depthBin]; + + // Concatenate the subtrees of shorter length + ByteString newTree = prefixesStack.pop(); + while (!prefixesStack.isEmpty() + && prefixesStack.peek().size() < binStart) { + ByteString left = prefixesStack.pop(); + newTree = new RopeByteString(left, newTree); + } + + // Concatenate the given string + newTree = new RopeByteString(newTree, byteString); + + // Continue concatenating until we land in an empty bin + while (!prefixesStack.isEmpty()) { + depthBin = getDepthBinForLength(newTree.size()); + binEnd = minLengthByDepth[depthBin + 1]; + if (prefixesStack.peek().size() < binEnd) { + ByteString left = prefixesStack.pop(); + newTree = new RopeByteString(left, newTree); + } else { + break; + } + } + prefixesStack.push(newTree); + } + } + + private int getDepthBinForLength(int length) { + int depth = Arrays.binarySearch(minLengthByDepth, length); + if (depth < 0) { + // It wasn't an exact match, so convert to the index of the containing + // fragment, which is one less even than the insertion point. + int insertionPoint = -(depth + 1); + depth = insertionPoint - 1; + } + + return depth; + } + } + + /** + * This class is a continuable tree traversal, which keeps the state + * information which would exist on the stack in a recursive traversal instead + * on a stack of "Bread Crumbs". The maximum depth of the stack in this + * iterator is the same as the depth of the tree being traversed. + * + *

This iterator is used to implement + * {@link RopeByteString#equalsFragments(ByteString)}. + */ + private static class PieceIterator implements Iterator { + + private final Deque breadCrumbs = + new ArrayDeque(minLengthByDepth.length); + private LiteralByteString next; + + private PieceIterator(ByteString root) { + next = getLeafByLeft(root); + } + + private LiteralByteString getLeafByLeft(ByteString root) { + ByteString pos = root; + while (pos instanceof RopeByteString) { + RopeByteString rbs = (RopeByteString) pos; + breadCrumbs.push(rbs); + pos = rbs.left; + } + return (LiteralByteString) pos; + } + + private LiteralByteString getNextNonEmptyLeaf() { + while (true) { + // Almost always, we go through this loop exactly once. However, if + // we discover an empty string in the rope, we toss it and try again. + if (breadCrumbs.isEmpty()) { + return null; + } else { + LiteralByteString result = getLeafByLeft(breadCrumbs.pop().right); + if (!result.isEmpty()) { + return result; + } + } + } + } + + public boolean hasNext() { + return next != null; + } + + /** + * Returns the next item and advances one {@code LiteralByteString}. + * + * @return next non-empty LiteralByteString or {@code null} + */ + public LiteralByteString next() { + if (next == null) { + throw new NoSuchElementException(); + } + LiteralByteString result = next; + next = getNextNonEmptyLeaf(); + return result; + } + + public void remove() { + throw new UnsupportedOperationException(); + } + } + + // ================================================================= + // ByteIterator + + @Override + public ByteIterator iterator() { + return new RopeByteIterator(); + } + + private class RopeByteIterator implements ByteString.ByteIterator { + + private final PieceIterator pieces; + private ByteIterator bytes; + int bytesRemaining; + + private RopeByteIterator() { + pieces = new PieceIterator(RopeByteString.this); + bytes = pieces.next().iterator(); + bytesRemaining = size(); + } + + public boolean hasNext() { + return (bytesRemaining > 0); + } + + public Byte next() { + return nextByte(); // Does not instantiate a Byte + } + + public byte nextByte() { + if (!bytes.hasNext()) { + bytes = pieces.next().iterator(); + } + --bytesRemaining; + return bytes.nextByte(); + } + + public void remove() { + throw new UnsupportedOperationException(); + } + } + + /** + * This class is the {@link RopeByteString} equivalent for + * {@link ByteArrayInputStream}. + */ + private class RopeInputStream extends InputStream { + // Iterates through the pieces of the rope + private PieceIterator pieceIterator; + // The current piece + private LiteralByteString currentPiece; + // The size of the current piece + private int currentPieceSize; + // The index of the next byte to read in the current piece + private int currentPieceIndex; + // The offset of the start of the current piece in the rope byte string + private int currentPieceOffsetInRope; + // Offset in the buffer at which user called mark(); + private int mark; + + public RopeInputStream() { + initialize(); + } + + @Override + public int read(byte b[], int offset, int length) { + if (b == null) { + throw new NullPointerException(); + } else if (offset < 0 || length < 0 || length > b.length - offset) { + throw new IndexOutOfBoundsException(); + } + return readSkipInternal(b, offset, length); + } + + @Override + public long skip(long length) { + if (length < 0) { + throw new IndexOutOfBoundsException(); + } else if (length > Integer.MAX_VALUE) { + length = Integer.MAX_VALUE; + } + return readSkipInternal(null, 0, (int) length); + } + + /** + * Internal implementation of read and skip. If b != null, then read the + * next {@code length} bytes into the buffer {@code b} at + * offset {@code offset}. If b == null, then skip the next {@code length) + * bytes. + *

+ * This method assumes that all error checking has already happened. + *

+ * Returns the actual number of bytes read or skipped. + */ + private int readSkipInternal(byte b[], int offset, int length) { + int bytesRemaining = length; + while (bytesRemaining > 0) { + advanceIfCurrentPieceFullyRead(); + if (currentPiece == null) { + if (bytesRemaining == length) { + // We didn't manage to read anything + return -1; + } + break; + } else { + // Copy the bytes from this piece. + int currentPieceRemaining = currentPieceSize - currentPieceIndex; + int count = Math.min(currentPieceRemaining, bytesRemaining); + if (b != null) { + currentPiece.copyTo(b, currentPieceIndex, offset, count); + offset += count; + } + currentPieceIndex += count; + bytesRemaining -= count; + } + } + // Return the number of bytes read. + return length - bytesRemaining; + } + + @Override + public int read() throws IOException { + advanceIfCurrentPieceFullyRead(); + if (currentPiece == null) { + return -1; + } else { + return currentPiece.byteAt(currentPieceIndex++) & 0xFF; + } + } + + @Override + public int available() throws IOException { + int bytesRead = currentPieceOffsetInRope + currentPieceIndex; + return RopeByteString.this.size() - bytesRead; + } + + @Override + public boolean markSupported() { + return true; + } + + @Override + public void mark(int readAheadLimit) { + // Set the mark to our position in the byte string + mark = currentPieceOffsetInRope + currentPieceIndex; + } + + @Override + public synchronized void reset() { + // Just reinitialize and skip the specified number of bytes. + initialize(); + readSkipInternal(null, 0, mark); + } + + /** Common initialization code used by both the constructor and reset() */ + private void initialize() { + pieceIterator = new PieceIterator(RopeByteString.this); + currentPiece = pieceIterator.next(); + currentPieceSize = currentPiece.size(); + currentPieceIndex = 0; + currentPieceOffsetInRope = 0; + } + + /** + * Skips to the next piece if we have read all the data in the current + * piece. Sets currentPiece to null if we have reached the end of the + * input. + */ + private void advanceIfCurrentPieceFullyRead() { + if (currentPiece != null && currentPieceIndex == currentPieceSize) { + // Generally, we can only go through this loop at most once, since + // empty strings can't end up in a rope. But better to test. + currentPieceOffsetInRope += currentPieceSize; + currentPieceIndex = 0; + if (pieceIterator.hasNext()) { + currentPiece = pieceIterator.next(); + currentPieceSize = currentPiece.size(); + } else { + currentPiece = null; + currentPieceSize = 0; + } + } + } + } +} diff --git a/java/src/main/java/com/google/protobuf/SingleFieldBuilder.java b/java/src/main/java/com/google/protobuf/SingleFieldBuilder.java index d4475f66..4bfc9f34 100644 --- a/java/src/main/java/com/google/protobuf/SingleFieldBuilder.java +++ b/java/src/main/java/com/google/protobuf/SingleFieldBuilder.java @@ -31,21 +31,21 @@ package com.google.protobuf; /** - * SingleFieldBuilder implements a structure that a protocol + * {@code SingleFieldBuilder} implements a structure that a protocol * message uses to hold a single field of another protocol message. It supports * the classical use case of setting an immutable {@link Message} as the value * of the field and is highly optimized around this. *
* It also supports the additional use case of setting a {@link Message.Builder} - * as the field and deferring conversion of that Builder - * to an immutable Message. In this way, it's possible to maintain - * a tree of Builder's that acts as a fully read/write data + * as the field and deferring conversion of that {@code Builder} + * to an immutable {@code Message}. In this way, it's possible to maintain + * a tree of {@code Builder}'s that acts as a fully read/write data * structure. *
* Logically, one can think of a tree of builders as converting the entire tree * to messages when build is called on the root or when any method is called * that desires a Message instead of a Builder. In terms of the implementation, - * the SingleFieldBuilder and RepeatedFieldBuilder + * the {@code SingleFieldBuilder} and {@code RepeatedFieldBuilder} * classes cache messages that were created so that messages only need to be * created when some change occured in its builder or a builder for one of its * descendants. @@ -99,7 +99,7 @@ public class SingleFieldBuilder /** * Get the message for the field. If the message is currently stored - * as a Builder, it is converted to a Message by + * as a {@code Builder}, it is converted to a {@code Message} by * calling {@link Message.Builder#buildPartial} on it. If no message has * been set, returns the default instance of the message. * diff --git a/java/src/main/java/com/google/protobuf/SmallSortedMap.java b/java/src/main/java/com/google/protobuf/SmallSortedMap.java index 1cf270f3..c6cad6af 100644 --- a/java/src/main/java/com/google/protobuf/SmallSortedMap.java +++ b/java/src/main/java/com/google/protobuf/SmallSortedMap.java @@ -51,14 +51,14 @@ import java.util.SortedMap; * remaining entries are stored in an overflow map. Iteration over the entries * in the map should be done as follows: * - * 

- * for (int i = 0; i < fieldMap.getNumArrayEntries(); i++) {
+ * 
   {@code
+ * for (int i = 0; i < fieldMap.getNumArrayEntries(); i++) {
  *   process(fieldMap.getArrayEntryAt(i));
  * }
- * for (Map.Entry<K, V> entry : fieldMap.getOverflowEntries()) {
+ * for (Map.Entry entry : fieldMap.getOverflowEntries()) {
  *   process(entry);
  * }
- * 

+ * }
* * The resulting iteration is in order of ascending field tag number. The * object returned by {@link #entrySet()} adheres to the same contract but is @@ -394,7 +394,7 @@ class SmallSortedMap, V> extends AbstractMap { /** * Entry implementation that implements Comparable in order to support - * binary search witin the entry array. Also checks mutability in + * binary search within the entry array. Also checks mutability in * {@link #setValue()}. */ private class Entry implements Map.Entry, Comparable { diff --git a/java/src/main/java/com/google/protobuf/TextFormat.java b/java/src/main/java/com/google/protobuf/TextFormat.java index d5fbdabf..ed462899 100644 --- a/java/src/main/java/com/google/protobuf/TextFormat.java +++ b/java/src/main/java/com/google/protobuf/TextFormat.java @@ -55,15 +55,18 @@ import java.util.regex.Pattern; public final class TextFormat { private TextFormat() {} - private static final Printer DEFAULT_PRINTER = new Printer(false); - private static final Printer SINGLE_LINE_PRINTER = new Printer(true); + private static final Printer DEFAULT_PRINTER = new Printer(); + private static final Printer SINGLE_LINE_PRINTER = + (new Printer()).setSingleLineMode(true); + private static final Printer UNICODE_PRINTER = + (new Printer()).setEscapeNonAscii(false); /** * Outputs a textual representation of the Protocol Message supplied into * the parameter output. (This representation is the new version of the * classic "ProtocolPrinter" output from the original Protocol Buffer system) */ - public static void print(final Message message, final Appendable output) + public static void print(final MessageOrBuilder message, final Appendable output) throws IOException { DEFAULT_PRINTER.print(message, new TextGenerator(output)); } @@ -79,7 +82,7 @@ public final class TextFormat { * Generates a human readable form of this message, useful for debugging and * other purposes, with no newline characters. */ - public static String shortDebugString(final Message message) { + public static String shortDebugString(final MessageOrBuilder message) { try { final StringBuilder sb = new StringBuilder(); SINGLE_LINE_PRINTER.print(message, new TextGenerator(sb)); @@ -109,7 +112,7 @@ public final class TextFormat { * Like {@code print()}, but writes directly to a {@code String} and * returns it. */ - public static String printToString(final Message message) { + public static String printToString(final MessageOrBuilder message) { try { final StringBuilder text = new StringBuilder(); print(message, text); @@ -133,6 +136,34 @@ public final class TextFormat { } } + /** + * Same as {@code printToString()}, except that non-ASCII characters + * in string type fields are not escaped in backslash+octals. + */ + public static String printToUnicodeString(final MessageOrBuilder message) { + try { + final StringBuilder text = new StringBuilder(); + UNICODE_PRINTER.print(message, new TextGenerator(text)); + return text.toString(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + /** + * Same as {@code printToString()}, except that non-ASCII characters + * in string type fields are not escaped in backslash+octals. + */ + public static String printToUnicodeString(final UnknownFieldSet fields) { + try { + final StringBuilder text = new StringBuilder(); + UNICODE_PRINTER.printUnknownFields(fields, new TextGenerator(text)); + return text.toString(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + public static void printField(final FieldDescriptor field, final Object value, final Appendable output) @@ -216,13 +247,26 @@ public final class TextFormat { /** Helper class for converting protobufs to text. */ private static final class Printer { /** Whether to omit newlines from the output. */ - final boolean singleLineMode; + boolean singleLineMode = false; + + /** Whether to escape non ASCII characters with backslash and octal. */ + boolean escapeNonAscii = true; + + private Printer() {} - private Printer(final boolean singleLineMode) { + /** Setter of singleLineMode */ + private Printer setSingleLineMode(boolean singleLineMode) { this.singleLineMode = singleLineMode; + return this; + } + + /** Setter of escapeNonAscii */ + private Printer setEscapeNonAscii(boolean escapeNonAscii) { + this.escapeNonAscii = escapeNonAscii; + return this; } - private void print(final Message message, final TextGenerator generator) + private void print(final MessageOrBuilder message, final TextGenerator generator) throws IOException { for (Map.Entry field : message.getAllFields().entrySet()) { @@ -339,7 +383,9 @@ public final class TextFormat { case STRING: generator.print("\""); - generator.print(escapeText((String) value)); + generator.print(escapeNonAscii ? + escapeText((String) value) : + (String) value); generator.print("\""); break; @@ -541,7 +587,7 @@ public final class TextFormat { private int previousLine = 0; private int previousColumn = 0; - // We use possesive quantifiers (*+ and ++) because otherwise the Java + // We use possessive quantifiers (*+ and ++) because otherwise the Java // regex matcher has stack overflows on large inputs. private static final Pattern WHITESPACE = Pattern.compile("(\\s|(#.*$))++", Pattern.MULTILINE); @@ -864,7 +910,7 @@ public final class TextFormat { public ParseException parseException(final String description) { // Note: People generally prefer one-based line and column numbers. return new ParseException( - (line + 1) + ":" + (column + 1) + ": " + description); + line + 1, column + 1, description); } /** @@ -875,7 +921,7 @@ public final class TextFormat { final String description) { // Note: People generally prefer one-based line and column numbers. return new ParseException( - (previousLine + 1) + ":" + (previousColumn + 1) + ": " + description); + previousLine + 1, previousColumn + 1, description); } /** @@ -900,8 +946,45 @@ public final class TextFormat { public static class ParseException extends IOException { private static final long serialVersionUID = 3196188060225107702L; + private final int line; + private final int column; + + /** Create a new instance, with -1 as the line and column numbers. */ public ParseException(final String message) { - super(message); + this(-1, -1, message); + } + + /** + * Create a new instance + * + * @param line the line number where the parse error occurred, + * using 1-offset. + * @param column the column number where the parser error occurred, + * using 1-offset. + */ + public ParseException(final int line, final int column, + final String message) { + super(Integer.toString(line) + ":" + column + ": " + message); + this.line = line; + this.column = column; + } + + /** + * Return the line where the parse exception occurred, or -1 when + * none is provided. The value is specified as 1-offset, so the first + * line is line 1. + */ + public int getLine() { + return line; + } + + /** + * Return the column where the parse exception occurred, or -1 when + * none is provided. The value is specified as 1-offset, so the first + * line is line 1. + */ + public int getColumn() { + return column; } } @@ -1073,7 +1156,7 @@ public final class TextFormat { mergeField(tokenizer, extensionRegistry, subBuilder); } - value = subBuilder.build(); + value = subBuilder.buildPartial(); } else { tokenizer.consume(":"); @@ -1212,7 +1295,7 @@ public final class TextFormat { */ static ByteString unescapeBytes(final CharSequence charString) throws InvalidEscapeSequenceException { - // First convert the Java characater sequence to UTF-8 bytes. + // First convert the Java character sequence to UTF-8 bytes. ByteString input = ByteString.copyFromUtf8(charString.toString()); // Then unescape certain byte sequences introduced by ASCII '\\'. The valid // escapes can all be expressed with ASCII characters, so it is safe to @@ -1349,7 +1432,7 @@ public final class TextFormat { /** * Parse a 32-bit signed integer from the text. Unlike the Java standard * {@code Integer.parseInt()}, this function recognizes the prefixes "0x" - * and "0" to signify hexidecimal and octal numbers, respectively. + * and "0" to signify hexadecimal and octal numbers, respectively. */ static int parseInt32(final String text) throws NumberFormatException { return (int) parseInteger(text, true, false); @@ -1358,7 +1441,7 @@ public final class TextFormat { /** * Parse a 32-bit unsigned integer from the text. Unlike the Java standard * {@code Integer.parseInt()}, this function recognizes the prefixes "0x" - * and "0" to signify hexidecimal and octal numbers, respectively. The + * and "0" to signify hexadecimal and octal numbers, respectively. The * result is coerced to a (signed) {@code int} when returned since Java has * no unsigned integer type. */ @@ -1369,7 +1452,7 @@ public final class TextFormat { /** * Parse a 64-bit signed integer from the text. Unlike the Java standard * {@code Integer.parseInt()}, this function recognizes the prefixes "0x" - * and "0" to signify hexidecimal and octal numbers, respectively. + * and "0" to signify hexadecimal and octal numbers, respectively. */ static long parseInt64(final String text) throws NumberFormatException { return parseInteger(text, true, true); @@ -1378,7 +1461,7 @@ public final class TextFormat { /** * Parse a 64-bit unsigned integer from the text. Unlike the Java standard * {@code Integer.parseInt()}, this function recognizes the prefixes "0x" - * and "0" to signify hexidecimal and octal numbers, respectively. The + * and "0" to signify hexadecimal and octal numbers, respectively. The * result is coerced to a (signed) {@code long} when returned since Java has * no unsigned long type. */ diff --git a/java/src/main/java/com/google/protobuf/UnknownFieldSet.java b/java/src/main/java/com/google/protobuf/UnknownFieldSet.java index 26a15d00..45e2e6e4 100644 --- a/java/src/main/java/com/google/protobuf/UnknownFieldSet.java +++ b/java/src/main/java/com/google/protobuf/UnknownFieldSet.java @@ -46,7 +46,7 @@ import java.util.TreeMap; * {@code UnknownFieldSet} is used to keep track of fields which were seen when * parsing a protocol message but whose field numbers or types are unrecognized. * This most frequently occurs when new fields are added to a message type - * and then messages containing those feilds are read by old software that was + * and then messages containing those fields are read by old software that was * compiled before the new types were added. * *

Every {@link Message} contains an {@code UnknownFieldSet} (and every @@ -468,7 +468,7 @@ public final class UnknownFieldSet implements MessageLite { /** * Parse a single field from {@code input} and merge it into this set. * @param tag The field's tag number, which was already parsed. - * @return {@code false} if the tag is an engroup tag. + * @return {@code false} if the tag is an end group tag. */ public boolean mergeFieldFrom(final int tag, final CodedInputStream input) throws IOException { @@ -950,4 +950,29 @@ public final class UnknownFieldSet implements MessageLite { } } } + + /** + * Parser to implement MessageLite interface. + */ + public static final class Parser extends AbstractParser { + public UnknownFieldSet parsePartialFrom( + CodedInputStream input, ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + Builder builder = newBuilder(); + try { + builder.mergeFrom(input); + } catch (InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(builder.buildPartial()); + } catch (IOException e) { + throw new InvalidProtocolBufferException(e.getMessage()) + .setUnfinishedMessage(builder.buildPartial()); + } + return builder.buildPartial(); + } + } + + private static final Parser PARSER = new Parser(); + public final Parser getParserForType() { + return PARSER; + } } diff --git a/java/src/main/java/com/google/protobuf/UnmodifiableLazyStringList.java b/java/src/main/java/com/google/protobuf/UnmodifiableLazyStringList.java index 83e5c796..f80f0968 100644 --- a/java/src/main/java/com/google/protobuf/UnmodifiableLazyStringList.java +++ b/java/src/main/java/com/google/protobuf/UnmodifiableLazyStringList.java @@ -32,6 +32,7 @@ package com.google.protobuf; import java.util.AbstractList; import java.util.RandomAccess; +import java.util.List; import java.util.ListIterator; import java.util.Iterator; @@ -143,4 +144,10 @@ public class UnmodifiableLazyStringList extends AbstractList } }; } + + @Override + public List getUnderlyingElements() { + // The returned value is already unmodifiable. + return list.getUnderlyingElements(); + } } diff --git a/java/src/main/java/com/google/protobuf/Utf8.java b/java/src/main/java/com/google/protobuf/Utf8.java new file mode 100644 index 00000000..388f7fc5 --- /dev/null +++ b/java/src/main/java/com/google/protobuf/Utf8.java @@ -0,0 +1,349 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// http://code.google.com/p/protobuf/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +/** + * A set of low-level, high-performance static utility methods related + * to the UTF-8 character encoding. This class has no dependencies + * outside of the core JDK libraries. + * + *

There are several variants of UTF-8. The one implemented by + * this class is the restricted definition of UTF-8 introduced in + * Unicode 3.1, which mandates the rejection of "overlong" byte + * sequences as well as rejection of 3-byte surrogate codepoint byte + * sequences. Note that the UTF-8 decoder included in Oracle's JDK + * has been modified to also reject "overlong" byte sequences, but (as + * of 2011) still accepts 3-byte surrogate codepoint byte sequences. + * + *

The byte sequences considered valid by this class are exactly + * those that can be roundtrip converted to Strings and back to bytes + * using the UTF-8 charset, without loss: 

 {@code
+ * Arrays.equals(bytes, new String(bytes, "UTF-8").getBytes("UTF-8"))
+ * }
+ * + *

See the Unicode Standard,
+ * Table 3-6. UTF-8 Bit Distribution,
+ * Table 3-7. Well Formed UTF-8 Byte Sequences. + * + *

This class supports decoding of partial byte sequences, so that the + * bytes in a complete UTF-8 byte sequences can be stored in multiple + * segments. Methods typically return {@link #MALFORMED} if the partial + * byte sequence is definitely not well-formed, {@link #COMPLETE} if it is + * well-formed in the absence of additional input, or if the byte sequence + * apparently terminated in the middle of a character, an opaque integer + * "state" value containing enough information to decode the character when + * passed to a subsequent invocation of a partial decoding method. + * + * @author martinrb@google.com (Martin Buchholz) + */ +final class Utf8 { + private Utf8() {} + + /** + * State value indicating that the byte sequence is well-formed and + * complete (no further bytes are needed to complete a character). + */ + public static final int COMPLETE = 0; + + /** + * State value indicating that the byte sequence is definitely not + * well-formed. + */ + public static final int MALFORMED = -1; + + // Other state values include the partial bytes of the incomplete + // character to be decoded in the simplest way: we pack the bytes + // into the state int in little-endian order. For example: + // + // int state = byte1 ^ (byte2 << 8) ^ (byte3 << 16); + // + // Such a state is unpacked thus (note the ~ operation for byte2 to + // undo byte1's sign-extension bits): + // + // int byte1 = (byte) state; + // int byte2 = (byte) ~(state >> 8); + // int byte3 = (byte) (state >> 16); + // + // We cannot store a zero byte in the state because it would be + // indistinguishable from the absence of a byte. But we don't need + // to, because partial bytes must always be negative. When building + // a state, we ensure that byte1 is negative and subsequent bytes + // are valid trailing bytes. + + /** + * Returns {@code true} if the given byte array is a well-formed + * UTF-8 byte sequence. + * + *

This is a convenience method, equivalent to a call to {@code + * isValidUtf8(bytes, 0, bytes.length)}. + */ + public static boolean isValidUtf8(byte[] bytes) { + return isValidUtf8(bytes, 0, bytes.length); + } + + /** + * Returns {@code true} if the given byte array slice is a + * well-formed UTF-8 byte sequence. The range of bytes to be + * checked extends from index {@code index}, inclusive, to {@code + * limit}, exclusive. + * + *

This is a convenience method, equivalent to {@code + * partialIsValidUtf8(bytes, index, limit) == Utf8.COMPLETE}. + */ + public static boolean isValidUtf8(byte[] bytes, int index, int limit) { + return partialIsValidUtf8(bytes, index, limit) == COMPLETE; + } + + /** + * Tells whether the given byte array slice is a well-formed, + * malformed, or incomplete UTF-8 byte sequence. The range of bytes + * to be checked extends from index {@code index}, inclusive, to + * {@code limit}, exclusive. + * + * @param state either {@link Utf8#COMPLETE} (if this is the initial decoding + * operation) or the value returned from a call to a partial decoding method + * for the previous bytes + * + * @return {@link #MALFORMED} if the partial byte sequence is + * definitely not well-formed, {@link #COMPLETE} if it is well-formed + * (no additional input needed), or if the byte sequence is + * "incomplete", i.e. apparently terminated in the middle of a character, + * an opaque integer "state" value containing enough information to + * decode the character when passed to a subsequent invocation of a + * partial decoding method. + */ + public static int partialIsValidUtf8( + int state, byte[] bytes, int index, int limit) { + if (state != COMPLETE) { + // The previous decoding operation was incomplete (or malformed). + // We look for a well-formed sequence consisting of bytes from + // the previous decoding operation (stored in state) together + // with bytes from the array slice. + // + // We expect such "straddler characters" to be rare. + + if (index >= limit) { // No bytes? No progress. + return state; + } + int byte1 = (byte) state; + // byte1 is never ASCII. + if (byte1 < (byte) 0xE0) { + // two-byte form + + // Simultaneously checks for illegal trailing-byte in + // leading position and overlong 2-byte form. + if (byte1 < (byte) 0xC2 || + // byte2 trailing-byte test + bytes[index++] > (byte) 0xBF) { + return MALFORMED; + } + } else if (byte1 < (byte) 0xF0) { + // three-byte form + + // Get byte2 from saved state or array + int byte2 = (byte) ~(state >> 8); + if (byte2 == 0) { + byte2 = bytes[index++]; + if (index >= limit) { + return incompleteStateFor(byte1, byte2); + } + } + if (byte2 > (byte) 0xBF || + // overlong? 5 most significant bits must not all be zero + (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) || + // illegal surrogate codepoint? + (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) || + // byte3 trailing-byte test + bytes[index++] > (byte) 0xBF) { + return MALFORMED; + } + } else { + // four-byte form + + // Get byte2 and byte3 from saved state or array + int byte2 = (byte) ~(state >> 8); + int byte3 = 0; + if (byte2 == 0) { + byte2 = bytes[index++]; + if (index >= limit) { + return incompleteStateFor(byte1, byte2); + } + } else { + byte3 = (byte) (state >> 16); + } + if (byte3 == 0) { + byte3 = bytes[index++]; + if (index >= limit) { + return incompleteStateFor(byte1, byte2, byte3); + } + } + + // If we were called with state == MALFORMED, then byte1 is 0xFF, + // which never occurs in well-formed UTF-8, and so we will return + // MALFORMED again below. + + if (byte2 > (byte) 0xBF || + // Check that 1 <= plane <= 16. Tricky optimized form of: + // if (byte1 > (byte) 0xF4 || + // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || + // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) + (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 || + // byte3 trailing-byte test + byte3 > (byte) 0xBF || + // byte4 trailing-byte test + bytes[index++] > (byte) 0xBF) { + return MALFORMED; + } + } + } + + return partialIsValidUtf8(bytes, index, limit); + } + + /** + * Tells whether the given byte array slice is a well-formed, + * malformed, or incomplete UTF-8 byte sequence. The range of bytes + * to be checked extends from index {@code index}, inclusive, to + * {@code limit}, exclusive. + * + *

This is a convenience method, equivalent to a call to {@code + * partialIsValidUtf8(Utf8.COMPLETE, bytes, index, limit)}. + * + * @return {@link #MALFORMED} if the partial byte sequence is + * definitely not well-formed, {@link #COMPLETE} if it is well-formed + * (no additional input needed), or if the byte sequence is + * "incomplete", i.e. apparently terminated in the middle of a character, + * an opaque integer "state" value containing enough information to + * decode the character when passed to a subsequent invocation of a + * partial decoding method. + */ + public static int partialIsValidUtf8( + byte[] bytes, int index, int limit) { + // Optimize for 100% ASCII. + // Hotspot loves small simple top-level loops like this. + while (index < limit && bytes[index] >= 0) { + index++; + } + + return (index >= limit) ? COMPLETE : + partialIsValidUtf8NonAscii(bytes, index, limit); + } + + private static int partialIsValidUtf8NonAscii( + byte[] bytes, int index, int limit) { + for (;;) { + int byte1, byte2; + + // Optimize for interior runs of ASCII bytes. + do { + if (index >= limit) { + return COMPLETE; + } + } while ((byte1 = bytes[index++]) >= 0); + + if (byte1 < (byte) 0xE0) { + // two-byte form + + if (index >= limit) { + return byte1; + } + + // Simultaneously checks for illegal trailing-byte in + // leading position and overlong 2-byte form. + if (byte1 < (byte) 0xC2 || + bytes[index++] > (byte) 0xBF) { + return MALFORMED; + } + } else if (byte1 < (byte) 0xF0) { + // three-byte form + + if (index >= limit - 1) { // incomplete sequence + return incompleteStateFor(bytes, index, limit); + } + if ((byte2 = bytes[index++]) > (byte) 0xBF || + // overlong? 5 most significant bits must not all be zero + (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) || + // check for illegal surrogate codepoints + (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) || + // byte3 trailing-byte test + bytes[index++] > (byte) 0xBF) { + return MALFORMED; + } + } else { + // four-byte form + + if (index >= limit - 2) { // incomplete sequence + return incompleteStateFor(bytes, index, limit); + } + if ((byte2 = bytes[index++]) > (byte) 0xBF || + // Check that 1 <= plane <= 16. Tricky optimized form of: + // if (byte1 > (byte) 0xF4 || + // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || + // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) + (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 || + // byte3 trailing-byte test + bytes[index++] > (byte) 0xBF || + // byte4 trailing-byte test + bytes[index++] > (byte) 0xBF) { + return MALFORMED; + } + } + } + } + + private static int incompleteStateFor(int byte1) { + return (byte1 > (byte) 0xF4) ? + MALFORMED : byte1; + } + + private static int incompleteStateFor(int byte1, int byte2) { + return (byte1 > (byte) 0xF4 || + byte2 > (byte) 0xBF) ? + MALFORMED : byte1 ^ (byte2 << 8); + } + + private static int incompleteStateFor(int byte1, int byte2, int byte3) { + return (byte1 > (byte) 0xF4 || + byte2 > (byte) 0xBF || + byte3 > (byte) 0xBF) ? + MALFORMED : byte1 ^ (byte2 << 8) ^ (byte3 << 16); + } + + private static int incompleteStateFor(byte[] bytes, int index, int limit) { + int byte1 = bytes[index - 1]; + switch (limit - index) { + case 0: return incompleteStateFor(byte1); + case 1: return incompleteStateFor(byte1, bytes[index]); + case 2: return incompleteStateFor(byte1, bytes[index], bytes[index + 1]); + default: throw new AssertionError(); + } + } +} diff --git a/java/src/main/java/com/google/protobuf/WireFormat.java b/java/src/main/java/com/google/protobuf/WireFormat.java index a30f2a3c..dd2d6310 100644 --- a/java/src/main/java/com/google/protobuf/WireFormat.java +++ b/java/src/main/java/com/google/protobuf/WireFormat.java @@ -146,7 +146,7 @@ public final class WireFormat { public boolean isPackable() { return true; } } - // Field numbers for feilds in MessageSet wire format. + // Field numbers for fields in MessageSet wire format. static final int MESSAGE_SET_ITEM = 1; static final int MESSAGE_SET_TYPE_ID = 2; static final int MESSAGE_SET_MESSAGE = 3; diff --git a/java/src/test/java/com/google/protobuf/AbstractMessageTest.java b/java/src/test/java/com/google/protobuf/AbstractMessageTest.java index d53ce8d7..3d05cb7d 100644 --- a/java/src/test/java/com/google/protobuf/AbstractMessageTest.java +++ b/java/src/test/java/com/google/protobuf/AbstractMessageTest.java @@ -30,6 +30,7 @@ package com.google.protobuf; +import com.google.protobuf.Descriptors.FieldDescriptor; import protobuf_unittest.UnittestOptimizeFor.TestOptimizedForSize; import protobuf_unittest.UnittestProto; import protobuf_unittest.UnittestProto.ForeignMessage; @@ -167,6 +168,13 @@ public class AbstractMessageTest extends TestCase { wrappedBuilder.setUnknownFields(unknownFields); return this; } + @Override + public Message.Builder getFieldBuilder(FieldDescriptor field) { + return wrappedBuilder.getFieldBuilder(field); + } + } + public Parser getParserForType() { + return wrappedMessage.getParserForType(); } } @@ -220,6 +228,34 @@ public class AbstractMessageTest extends TestCase { TestUtil.assertAllFieldsSet((TestAllTypes) message.wrappedMessage); } + public void testParsingUninitialized() throws Exception { + TestRequiredForeign.Builder builder = TestRequiredForeign.newBuilder(); + builder.getOptionalMessageBuilder().setDummy2(10); + ByteString bytes = builder.buildPartial().toByteString(); + Message.Builder abstractMessageBuilder = + new AbstractMessageWrapper.Builder(TestRequiredForeign.newBuilder()); + // mergeFrom() should not throw initialization error. + abstractMessageBuilder.mergeFrom(bytes).buildPartial(); + try { + abstractMessageBuilder.mergeFrom(bytes).build(); + fail(); + } catch (UninitializedMessageException ex) { + // pass + } + + // test DynamicMessage directly. + Message.Builder dynamicMessageBuilder = DynamicMessage.newBuilder( + TestRequiredForeign.getDescriptor()); + // mergeFrom() should not throw initialization error. + dynamicMessageBuilder.mergeFrom(bytes).buildPartial(); + try { + dynamicMessageBuilder.mergeFrom(bytes).build(); + fail(); + } catch (UninitializedMessageException ex) { + // pass + } + } + public void testPackedSerialization() throws Exception { Message abstractMessage = new AbstractMessageWrapper(TestUtil.getPackedSet()); @@ -298,12 +334,16 @@ public class AbstractMessageTest extends TestCase { new AbstractMessageWrapper.Builder(builder); assertFalse(abstractBuilder.isInitialized()); + assertEquals("a, b, c", abstractBuilder.getInitializationErrorString()); builder.setA(1); assertFalse(abstractBuilder.isInitialized()); + assertEquals("b, c", abstractBuilder.getInitializationErrorString()); builder.setB(1); assertFalse(abstractBuilder.isInitialized()); + assertEquals("c", abstractBuilder.getInitializationErrorString()); builder.setC(1); assertTrue(abstractBuilder.isInitialized()); + assertEquals("", abstractBuilder.getInitializationErrorString()); } public void testForeignIsInitialized() throws Exception { @@ -312,18 +352,27 @@ public class AbstractMessageTest extends TestCase { new AbstractMessageWrapper.Builder(builder); assertTrue(abstractBuilder.isInitialized()); + assertEquals("", abstractBuilder.getInitializationErrorString()); builder.setOptionalMessage(TEST_REQUIRED_UNINITIALIZED); assertFalse(abstractBuilder.isInitialized()); + assertEquals( + "optional_message.a, optional_message.b, optional_message.c", + abstractBuilder.getInitializationErrorString()); builder.setOptionalMessage(TEST_REQUIRED_INITIALIZED); assertTrue(abstractBuilder.isInitialized()); + assertEquals("", abstractBuilder.getInitializationErrorString()); builder.addRepeatedMessage(TEST_REQUIRED_UNINITIALIZED); assertFalse(abstractBuilder.isInitialized()); + assertEquals( + "repeated_message[0].a, repeated_message[0].b, repeated_message[0].c", + abstractBuilder.getInitializationErrorString()); builder.setRepeatedMessage(0, TEST_REQUIRED_INITIALIZED); assertTrue(abstractBuilder.isInitialized()); + assertEquals("", abstractBuilder.getInitializationErrorString()); } // ----------------------------------------------------------------- @@ -421,7 +470,7 @@ public class AbstractMessageTest extends TestCase { /** - * Asserts that the given proto has symetric equals and hashCode methods. + * Asserts that the given proto has symmetric equals and hashCode methods. */ private void checkEqualsIsConsistent(Message message) { // Object should be equal to itself. diff --git a/java/src/test/java/com/google/protobuf/BoundedByteStringTest.java b/java/src/test/java/com/google/protobuf/BoundedByteStringTest.java new file mode 100644 index 00000000..20fa2dff --- /dev/null +++ b/java/src/test/java/com/google/protobuf/BoundedByteStringTest.java @@ -0,0 +1,68 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// http://code.google.com/p/protobuf/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import java.io.UnsupportedEncodingException; + +/** + * This class tests {@link BoundedByteString}, which extends {@link LiteralByteString}, + * by inheriting the tests from {@link LiteralByteStringTest}. The only method which + * is strange enough that it needs to be overridden here is {@link #testToString()}. + * + * @author carlanton@google.com (Carl Haverl) + */ +public class BoundedByteStringTest extends LiteralByteStringTest { + + @Override + protected void setUp() throws Exception { + classUnderTest = "BoundedByteString"; + byte[] sourceBytes = ByteStringTest.getTestBytes(2341, 11337766L); + int from = 100; + int to = sourceBytes.length - 100; + stringUnderTest = ByteString.copyFrom(sourceBytes).substring(from, to); + referenceBytes = new byte[to - from]; + System.arraycopy(sourceBytes, from, referenceBytes, 0, to - from); + expectedHashCode = 727575887; + } + + @Override + public void testToString() throws UnsupportedEncodingException { + String testString = "I love unicode \u1234\u5678 characters"; + LiteralByteString unicode = new LiteralByteString(testString.getBytes(UTF_8)); + ByteString chopped = unicode.substring(2, unicode.size() - 6); + assertEquals(classUnderTest + ".substring() must have the expected type", + classUnderTest, getActualClassName(chopped)); + + String roundTripString = chopped.toString(UTF_8); + assertEquals(classUnderTest + " unicode bytes must match", + testString.substring(2, testString.length() - 6), roundTripString); + } +} diff --git a/java/src/test/java/com/google/protobuf/ByteStringTest.java b/java/src/test/java/com/google/protobuf/ByteStringTest.java new file mode 100644 index 00000000..7a1d6823 --- /dev/null +++ b/java/src/test/java/com/google/protobuf/ByteStringTest.java @@ -0,0 +1,692 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// http://code.google.com/p/protobuf/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import com.google.protobuf.ByteString.Output; + +import junit.framework.TestCase; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Random; + +/** + * Test methods with implementations in {@link ByteString}, plus do some top-level "integration" + * tests. + * + * @author carlanton@google.com (Carl Haverl) + */ +public class ByteStringTest extends TestCase { + + private static final String UTF_16 = "UTF-16"; + + static byte[] getTestBytes(int size, long seed) { + Random random = new Random(seed); + byte[] result = new byte[size]; + random.nextBytes(result); + return result; + } + + private byte[] getTestBytes(int size) { + return getTestBytes(size, 445566L); + } + + private byte[] getTestBytes() { + return getTestBytes(1000); + } + + // Compare the entire left array with a subset of the right array. + private boolean isArrayRange(byte[] left, byte[] right, int rightOffset, int length) { + boolean stillEqual = (left.length == length); + for (int i = 0; (stillEqual && i < length); ++i) { + stillEqual = (left[i] == right[rightOffset + i]); + } + return stillEqual; + } + + // Returns true only if the given two arrays have identical contents. + private boolean isArray(byte[] left, byte[] right) { + return left.length == right.length && isArrayRange(left, right, 0, left.length); + } + + public void testSubstring_BeginIndex() { + byte[] bytes = getTestBytes(); + ByteString substring = ByteString.copyFrom(bytes).substring(500); + assertTrue("substring must contain the tail of the string", + isArrayRange(substring.toByteArray(), bytes, 500, bytes.length - 500)); + } + + public void testCopyFrom_BytesOffsetSize() { + byte[] bytes = getTestBytes(); + ByteString byteString = ByteString.copyFrom(bytes, 500, 200); + assertTrue("copyFrom sub-range must contain the expected bytes", + isArrayRange(byteString.toByteArray(), bytes, 500, 200)); + } + + public void testCopyFrom_Bytes() { + byte[] bytes = getTestBytes(); + ByteString byteString = ByteString.copyFrom(bytes); + assertTrue("copyFrom must contain the expected bytes", + isArray(byteString.toByteArray(), bytes)); + } + + public void testCopyFrom_ByteBufferSize() { + byte[] bytes = getTestBytes(); + ByteBuffer byteBuffer = ByteBuffer.allocate(bytes.length); + byteBuffer.put(bytes); + byteBuffer.position(500); + ByteString byteString = ByteString.copyFrom(byteBuffer, 200); + assertTrue("copyFrom byteBuffer sub-range must contain the expected bytes", + isArrayRange(byteString.toByteArray(), bytes, 500, 200)); + } + + public void testCopyFrom_ByteBuffer() { + byte[] bytes = getTestBytes(); + ByteBuffer byteBuffer = ByteBuffer.allocate(bytes.length); + byteBuffer.put(bytes); + byteBuffer.position(500); + ByteString byteString = ByteString.copyFrom(byteBuffer); + assertTrue("copyFrom byteBuffer sub-range must contain the expected bytes", + isArrayRange(byteString.toByteArray(), bytes, 500, bytes.length - 500)); + } + + public void testCopyFrom_StringEncoding() throws UnsupportedEncodingException { + String testString = "I love unicode \u1234\u5678 characters"; + ByteString byteString = ByteString.copyFrom(testString, UTF_16); + byte[] testBytes = testString.getBytes(UTF_16); + assertTrue("copyFrom string must respect the charset", + isArrayRange(byteString.toByteArray(), testBytes, 0, testBytes.length)); + } + + public void testCopyFrom_Utf8() throws UnsupportedEncodingException { + String testString = "I love unicode \u1234\u5678 characters"; + ByteString byteString = ByteString.copyFromUtf8(testString); + byte[] testBytes = testString.getBytes("UTF-8"); + assertTrue("copyFromUtf8 string must respect the charset", + isArrayRange(byteString.toByteArray(), testBytes, 0, testBytes.length)); + } + + public void testCopyFrom_Iterable() { + byte[] testBytes = getTestBytes(77777, 113344L); + final List pieces = makeConcretePieces(testBytes); + // Call copyFrom() on a Collection + ByteString byteString = ByteString.copyFrom(pieces); + assertTrue("copyFrom a List must contain the expected bytes", + isArrayRange(byteString.toByteArray(), testBytes, 0, testBytes.length)); + // Call copyFrom on an iteration that's not a collection + ByteString byteStringAlt = ByteString.copyFrom(new Iterable() { + public Iterator iterator() { + return pieces.iterator(); + } + }); + assertEquals("copyFrom from an Iteration must contain the expected bytes", + byteString, byteStringAlt); + } + + public void testCopyTo_TargetOffset() { + byte[] bytes = getTestBytes(); + ByteString byteString = ByteString.copyFrom(bytes); + byte[] target = new byte[bytes.length + 1000]; + byteString.copyTo(target, 400); + assertTrue("copyFrom byteBuffer sub-range must contain the expected bytes", + isArrayRange(bytes, target, 400, bytes.length)); + } + + public void testReadFrom_emptyStream() throws IOException { + ByteString byteString = + ByteString.readFrom(new ByteArrayInputStream(new byte[0])); + assertSame("reading an empty stream must result in the EMPTY constant " + + "byte string", ByteString.EMPTY, byteString); + } + + public void testReadFrom_smallStream() throws IOException { + assertReadFrom(getTestBytes(10)); + } + + public void testReadFrom_mutating() throws IOException { + byte[] capturedArray = null; + EvilInputStream eis = new EvilInputStream(); + ByteString byteString = ByteString.readFrom(eis); + + capturedArray = eis.capturedArray; + byte[] originalValue = byteString.toByteArray(); + for (int x = 0; x < capturedArray.length; ++x) { + capturedArray[x] = (byte) 0; + } + + byte[] newValue = byteString.toByteArray(); + assertTrue("copyFrom byteBuffer must not grant access to underlying array", + Arrays.equals(originalValue, newValue)); + } + + // Tests sizes that are near the rope copy-out threshold. + public void testReadFrom_mediumStream() throws IOException { + assertReadFrom(getTestBytes(ByteString.CONCATENATE_BY_COPY_SIZE - 1)); + assertReadFrom(getTestBytes(ByteString.CONCATENATE_BY_COPY_SIZE)); + assertReadFrom(getTestBytes(ByteString.CONCATENATE_BY_COPY_SIZE + 1)); + assertReadFrom(getTestBytes(200)); + } + + // Tests sizes that are over multi-segment rope threshold. + public void testReadFrom_largeStream() throws IOException { + assertReadFrom(getTestBytes(0x100)); + assertReadFrom(getTestBytes(0x101)); + assertReadFrom(getTestBytes(0x110)); + assertReadFrom(getTestBytes(0x1000)); + assertReadFrom(getTestBytes(0x1001)); + assertReadFrom(getTestBytes(0x1010)); + assertReadFrom(getTestBytes(0x10000)); + assertReadFrom(getTestBytes(0x10001)); + assertReadFrom(getTestBytes(0x10010)); + } + + // Tests sizes that are near the read buffer size. + public void testReadFrom_byteBoundaries() throws IOException { + final int min = ByteString.MIN_READ_FROM_CHUNK_SIZE; + final int max = ByteString.MAX_READ_FROM_CHUNK_SIZE; + + assertReadFrom(getTestBytes(min - 1)); + assertReadFrom(getTestBytes(min)); + assertReadFrom(getTestBytes(min + 1)); + + assertReadFrom(getTestBytes(min * 2 - 1)); + assertReadFrom(getTestBytes(min * 2)); + assertReadFrom(getTestBytes(min * 2 + 1)); + + assertReadFrom(getTestBytes(min * 4 - 1)); + assertReadFrom(getTestBytes(min * 4)); + assertReadFrom(getTestBytes(min * 4 + 1)); + + assertReadFrom(getTestBytes(min * 8 - 1)); + assertReadFrom(getTestBytes(min * 8)); + assertReadFrom(getTestBytes(min * 8 + 1)); + + assertReadFrom(getTestBytes(max - 1)); + assertReadFrom(getTestBytes(max)); + assertReadFrom(getTestBytes(max + 1)); + + assertReadFrom(getTestBytes(max * 2 - 1)); + assertReadFrom(getTestBytes(max * 2)); + assertReadFrom(getTestBytes(max * 2 + 1)); + } + + // Tests that IOExceptions propagate through ByteString.readFrom(). + public void testReadFrom_IOExceptions() { + try { + ByteString.readFrom(new FailStream()); + fail("readFrom must throw the underlying IOException"); + + } catch (IOException e) { + assertEquals("readFrom must throw the expected exception", + "synthetic failure", e.getMessage()); + } + } + + // Tests that ByteString.readFrom works with streams that don't + // always fill their buffers. + public void testReadFrom_reluctantStream() throws IOException { + final byte[] data = getTestBytes(0x1000); + + ByteString byteString = ByteString.readFrom(new ReluctantStream(data)); + assertTrue("readFrom byte stream must contain the expected bytes", + isArray(byteString.toByteArray(), data)); + + // Same test as above, but with some specific chunk sizes. + assertReadFromReluctantStream(data, 100); + assertReadFromReluctantStream(data, 248); + assertReadFromReluctantStream(data, 249); + assertReadFromReluctantStream(data, 250); + assertReadFromReluctantStream(data, 251); + assertReadFromReluctantStream(data, 0x1000); + assertReadFromReluctantStream(data, 0x1001); + } + + // Fails unless ByteString.readFrom reads the bytes correctly from a + // reluctant stream with the given chunkSize parameter. + private void assertReadFromReluctantStream(byte[] bytes, int chunkSize) + throws IOException { + ByteString b = ByteString.readFrom(new ReluctantStream(bytes), chunkSize); + assertTrue("readFrom byte stream must contain the expected bytes", + isArray(b.toByteArray(), bytes)); + } + + // Tests that ByteString.readFrom works with streams that implement + // available(). + public void testReadFrom_available() throws IOException { + final byte[] data = getTestBytes(0x1001); + + ByteString byteString = ByteString.readFrom(new AvailableStream(data)); + assertTrue("readFrom byte stream must contain the expected bytes", + isArray(byteString.toByteArray(), data)); + } + + // Fails unless ByteString.readFrom reads the bytes correctly. + private void assertReadFrom(byte[] bytes) throws IOException { + ByteString byteString = + ByteString.readFrom(new ByteArrayInputStream(bytes)); + assertTrue("readFrom byte stream must contain the expected bytes", + isArray(byteString.toByteArray(), bytes)); + } + + // A stream that fails when read. + private static final class FailStream extends InputStream { + @Override public int read() throws IOException { + throw new IOException("synthetic failure"); + } + } + + // A stream that simulates blocking by only producing 250 characters + // per call to read(byte[]). + private static class ReluctantStream extends InputStream { + protected final byte[] data; + protected int pos = 0; + + public ReluctantStream(byte[] data) { + this.data = data; + } + + @Override public int read() { + if (pos == data.length) { + return -1; + } else { + return data[pos++]; + } + } + + @Override public int read(byte[] buf) { + return read(buf, 0, buf.length); + } + + @Override public int read(byte[] buf, int offset, int size) { + if (pos == data.length) { + return -1; + } + int count = Math.min(Math.min(size, data.length - pos), 250); + System.arraycopy(data, pos, buf, offset, count); + pos += count; + return count; + } + } + + // Same as above, but also implements available(). + private static final class AvailableStream extends ReluctantStream { + public AvailableStream(byte[] data) { + super(data); + } + + @Override public int available() { + return Math.min(250, data.length - pos); + } + } + + // A stream which exposes the byte array passed into read(byte[], int, int). + private static class EvilInputStream extends InputStream { + public byte[] capturedArray = null; + + @Override + public int read(byte[] buf, int off, int len) { + if (capturedArray != null) { + return -1; + } else { + capturedArray = buf; + for (int x = 0; x < len; ++x) { + buf[x] = (byte) x; + } + return len; + } + } + + @Override + public int read() { + // Purposefully do nothing. + return -1; + } + } + + // A stream which exposes the byte array passed into write(byte[], int, int). + private static class EvilOutputStream extends OutputStream { + public byte[] capturedArray = null; + + @Override + public void write(byte[] buf, int off, int len) { + if (capturedArray == null) { + capturedArray = buf; + } + } + + @Override + public void write(int ignored) { + // Purposefully do nothing. + } + } + + public void testToStringUtf8() throws UnsupportedEncodingException { + String testString = "I love unicode \u1234\u5678 characters"; + byte[] testBytes = testString.getBytes("UTF-8"); + ByteString byteString = ByteString.copyFrom(testBytes); + assertEquals("copyToStringUtf8 must respect the charset", + testString, byteString.toStringUtf8()); + } + + public void testNewOutput_InitialCapacity() throws IOException { + byte[] bytes = getTestBytes(); + ByteString.Output output = ByteString.newOutput(bytes.length + 100); + output.write(bytes); + ByteString byteString = output.toByteString(); + assertTrue( + "String built from newOutput(int) must contain the expected bytes", + isArrayRange(bytes, byteString.toByteArray(), 0, bytes.length)); + } + + // Test newOutput() using a variety of buffer sizes and a variety of (fixed) + // write sizes + public void testNewOutput_ArrayWrite() throws IOException { + byte[] bytes = getTestBytes(); + int length = bytes.length; + int[] bufferSizes = {128, 256, length / 2, length - 1, length, length + 1, + 2 * length, 3 * length}; + int[] writeSizes = {1, 4, 5, 7, 23, bytes.length}; + + for (int bufferSize : bufferSizes) { + for (int writeSize : writeSizes) { + // Test writing the entire output writeSize bytes at a time. + ByteString.Output output = ByteString.newOutput(bufferSize); + for (int i = 0; i < length; i += writeSize) { + output.write(bytes, i, Math.min(writeSize, length - i)); + } + ByteString byteString = output.toByteString(); + assertTrue("String built from newOutput() must contain the expected bytes", + isArrayRange(bytes, byteString.toByteArray(), 0, bytes.length)); + } + } + } + + // Test newOutput() using a variety of buffer sizes, but writing all the + // characters using write(byte); + public void testNewOutput_WriteChar() throws IOException { + byte[] bytes = getTestBytes(); + int length = bytes.length; + int[] bufferSizes = {0, 1, 128, 256, length / 2, + length - 1, length, length + 1, + 2 * length, 3 * length}; + for (int bufferSize : bufferSizes) { + ByteString.Output output = ByteString.newOutput(bufferSize); + for (byte byteValue : bytes) { + output.write(byteValue); + } + ByteString byteString = output.toByteString(); + assertTrue("String built from newOutput() must contain the expected bytes", + isArrayRange(bytes, byteString.toByteArray(), 0, bytes.length)); + } + } + + // Test newOutput() in which we write the bytes using a variety of methods + // and sizes, and in which we repeatedly call toByteString() in the middle. + public void testNewOutput_Mixed() throws IOException { + Random rng = new Random(1); + byte[] bytes = getTestBytes(); + int length = bytes.length; + int[] bufferSizes = {0, 1, 128, 256, length / 2, + length - 1, length, length + 1, + 2 * length, 3 * length}; + + for (int bufferSize : bufferSizes) { + // Test writing the entire output using a mixture of write sizes and + // methods; + ByteString.Output output = ByteString.newOutput(bufferSize); + int position = 0; + while (position < bytes.length) { + if (rng.nextBoolean()) { + int count = 1 + rng.nextInt(bytes.length - position); + output.write(bytes, position, count); + position += count; + } else { + output.write(bytes[position]); + position++; + } + assertEquals("size() returns the right value", position, output.size()); + assertTrue("newOutput() substring must have correct bytes", + isArrayRange(output.toByteString().toByteArray(), + bytes, 0, position)); + } + ByteString byteString = output.toByteString(); + assertTrue("String built from newOutput() must contain the expected bytes", + isArrayRange(bytes, byteString.toByteArray(), 0, bytes.length)); + } + } + + public void testNewOutputEmpty() throws IOException { + // Make sure newOutput() correctly builds empty byte strings + ByteString byteString = ByteString.newOutput().toByteString(); + assertEquals(ByteString.EMPTY, byteString); + } + + public void testNewOutput_Mutating() throws IOException { + Output os = ByteString.newOutput(5); + os.write(new byte[] {1, 2, 3, 4, 5}); + EvilOutputStream eos = new EvilOutputStream(); + os.writeTo(eos); + byte[] capturedArray = eos.capturedArray; + ByteString byteString = os.toByteString(); + byte[] oldValue = byteString.toByteArray(); + Arrays.fill(capturedArray, (byte) 0); + byte[] newValue = byteString.toByteArray(); + assertTrue("Output must not provide access to the underlying byte array", + Arrays.equals(oldValue, newValue)); + } + + public void testNewCodedBuilder() throws IOException { + byte[] bytes = getTestBytes(); + ByteString.CodedBuilder builder = ByteString.newCodedBuilder(bytes.length); + builder.getCodedOutput().writeRawBytes(bytes); + ByteString byteString = builder.build(); + assertTrue("String built from newCodedBuilder() must contain the expected bytes", + isArrayRange(bytes, byteString.toByteArray(), 0, bytes.length)); + } + + public void testSubstringParity() { + byte[] bigBytes = getTestBytes(2048 * 1024, 113344L); + int start = 512 * 1024 - 3333; + int end = 512 * 1024 + 7777; + ByteString concreteSubstring = ByteString.copyFrom(bigBytes).substring(start, end); + boolean ok = true; + for (int i = start; ok && i < end; ++i) { + ok = (bigBytes[i] == concreteSubstring.byteAt(i - start)); + } + assertTrue("Concrete substring didn't capture the right bytes", ok); + + ByteString literalString = ByteString.copyFrom(bigBytes, start, end - start); + assertTrue("Substring must be equal to literal string", + concreteSubstring.equals(literalString)); + assertEquals("Substring must have same hashcode as literal string", + literalString.hashCode(), concreteSubstring.hashCode()); + } + + public void testCompositeSubstring() { + byte[] referenceBytes = getTestBytes(77748, 113344L); + + List pieces = makeConcretePieces(referenceBytes); + ByteString listString = ByteString.copyFrom(pieces); + + int from = 1000; + int to = 40000; + ByteString compositeSubstring = listString.substring(from, to); + byte[] substringBytes = compositeSubstring.toByteArray(); + boolean stillEqual = true; + for (int i = 0; stillEqual && i < to - from; ++i) { + stillEqual = referenceBytes[from + i] == substringBytes[i]; + } + assertTrue("Substring must return correct bytes", stillEqual); + + stillEqual = true; + for (int i = 0; stillEqual && i < to - from; ++i) { + stillEqual = referenceBytes[from + i] == compositeSubstring.byteAt(i); + } + assertTrue("Substring must support byteAt() correctly", stillEqual); + + ByteString literalSubstring = ByteString.copyFrom(referenceBytes, from, to - from); + assertTrue("Composite substring must equal a literal substring over the same bytes", + compositeSubstring.equals(literalSubstring)); + assertTrue("Literal substring must equal a composite substring over the same bytes", + literalSubstring.equals(compositeSubstring)); + + assertEquals("We must get the same hashcodes for composite and literal substrings", + literalSubstring.hashCode(), compositeSubstring.hashCode()); + + assertFalse("We can't be equal to a proper substring", + compositeSubstring.equals(literalSubstring.substring(0, literalSubstring.size() - 1))); + } + + public void testCopyFromList() { + byte[] referenceBytes = getTestBytes(77748, 113344L); + ByteString literalString = ByteString.copyFrom(referenceBytes); + + List pieces = makeConcretePieces(referenceBytes); + ByteString listString = ByteString.copyFrom(pieces); + + assertTrue("Composite string must be equal to literal string", + listString.equals(literalString)); + assertEquals("Composite string must have same hashcode as literal string", + literalString.hashCode(), listString.hashCode()); + } + + public void testConcat() { + byte[] referenceBytes = getTestBytes(77748, 113344L); + ByteString literalString = ByteString.copyFrom(referenceBytes); + + List pieces = makeConcretePieces(referenceBytes); + + Iterator iter = pieces.iterator(); + ByteString concatenatedString = iter.next(); + while (iter.hasNext()) { + concatenatedString = concatenatedString.concat(iter.next()); + } + + assertTrue("Concatenated string must be equal to literal string", + concatenatedString.equals(literalString)); + assertEquals("Concatenated string must have same hashcode as literal string", + literalString.hashCode(), concatenatedString.hashCode()); + } + + /** + * Test the Rope implementation can deal with Empty nodes, even though we + * guard against them. See also {@link LiteralByteStringTest#testConcat_empty()}. + */ + public void testConcat_empty() { + byte[] referenceBytes = getTestBytes(7748, 113344L); + ByteString literalString = ByteString.copyFrom(referenceBytes); + + ByteString duo = RopeByteString.newInstanceForTest(literalString, literalString); + ByteString temp = RopeByteString.newInstanceForTest( + RopeByteString.newInstanceForTest(literalString, ByteString.EMPTY), + RopeByteString.newInstanceForTest(ByteString.EMPTY, literalString)); + ByteString quintet = RopeByteString.newInstanceForTest(temp, ByteString.EMPTY); + + assertTrue("String with concatenated nulls must equal simple concatenate", + duo.equals(quintet)); + assertEquals("String with concatenated nulls have same hashcode as simple concatenate", + duo.hashCode(), quintet.hashCode()); + + ByteString.ByteIterator duoIter = duo.iterator(); + ByteString.ByteIterator quintetIter = quintet.iterator(); + boolean stillEqual = true; + while (stillEqual && quintetIter.hasNext()) { + stillEqual = (duoIter.nextByte() == quintetIter.nextByte()); + } + assertTrue("We must get the same characters by iterating", stillEqual); + assertFalse("Iterator must be exhausted", duoIter.hasNext()); + try { + duoIter.nextByte(); + fail("Should have thrown an exception."); + } catch (NoSuchElementException e) { + // This is success + } + try { + quintetIter.nextByte(); + fail("Should have thrown an exception."); + } catch (NoSuchElementException e) { + // This is success + } + + // Test that even if we force empty strings in as rope leaves in this + // configuration, we always get a (possibly Bounded) LiteralByteString + // for a length 1 substring. + // + // It is possible, using the testing factory method to create deeply nested + // trees of empty leaves, to make a string that will fail this test. + for (int i = 1; i < duo.size(); ++i) { + assertTrue("Substrings of size() < 2 must not be RopeByteStrings", + duo.substring(i - 1, i) instanceof LiteralByteString); + } + for (int i = 1; i < quintet.size(); ++i) { + assertTrue("Substrings of size() < 2 must not be RopeByteStrings", + quintet.substring(i - 1, i) instanceof LiteralByteString); + } + } + + public void testStartsWith() { + byte[] bytes = getTestBytes(1000, 1234L); + ByteString string = ByteString.copyFrom(bytes); + ByteString prefix = ByteString.copyFrom(bytes, 0, 500); + ByteString suffix = ByteString.copyFrom(bytes, 400, 600); + assertTrue(string.startsWith(ByteString.EMPTY)); + assertTrue(string.startsWith(string)); + assertTrue(string.startsWith(prefix)); + assertFalse(string.startsWith(suffix)); + assertFalse(prefix.startsWith(suffix)); + assertFalse(suffix.startsWith(prefix)); + assertFalse(ByteString.EMPTY.startsWith(prefix)); + assertTrue(ByteString.EMPTY.startsWith(ByteString.EMPTY)); + } + + static List makeConcretePieces(byte[] referenceBytes) { + List pieces = new ArrayList(); + // Starting length should be small enough that we'll do some concatenating by + // copying if we just concatenate all these pieces together. + for (int start = 0, length = 16; start < referenceBytes.length; start += length) { + length = (length << 1) - 1; + if (start + length > referenceBytes.length) { + length = referenceBytes.length - start; + } + pieces.add(ByteString.copyFrom(referenceBytes, start, length)); + } + return pieces; + } +} diff --git a/java/src/test/java/com/google/protobuf/CodedOutputStreamTest.java b/java/src/test/java/com/google/protobuf/CodedOutputStreamTest.java index 74476e36..86255029 100644 --- a/java/src/test/java/com/google/protobuf/CodedOutputStreamTest.java +++ b/java/src/test/java/com/google/protobuf/CodedOutputStreamTest.java @@ -30,8 +30,10 @@ package com.google.protobuf; +import protobuf_unittest.UnittestProto.SparseEnumMessage; import protobuf_unittest.UnittestProto.TestAllTypes; import protobuf_unittest.UnittestProto.TestPackedTypes; +import protobuf_unittest.UnittestProto.TestSparseEnum; import junit.framework.TestCase; @@ -302,17 +304,14 @@ public class CodedOutputStreamTest extends TestCase { } /** Test writing a message containing a negative enum value. This used to - * fail because the size was not properly computed as a sign-extended varint. */ + * fail because the size was not properly computed as a sign-extended varint. + */ public void testWriteMessageWithNegativeEnumValue() throws Exception { - protobuf_unittest.UnittestProto.SparseEnumMessage message = - protobuf_unittest.UnittestProto.SparseEnumMessage.newBuilder() - .setSparseEnum(protobuf_unittest.UnittestProto.TestSparseEnum.SPARSE_E) - .build(); + SparseEnumMessage message = SparseEnumMessage.newBuilder() + .setSparseEnum(TestSparseEnum.SPARSE_E) .build(); assertTrue(message.getSparseEnum().getNumber() < 0); byte[] rawBytes = message.toByteArray(); - protobuf_unittest.UnittestProto.SparseEnumMessage message2 = - protobuf_unittest.UnittestProto.SparseEnumMessage.parseFrom(rawBytes); - assertEquals(protobuf_unittest.UnittestProto.TestSparseEnum.SPARSE_E, - message2.getSparseEnum()); + SparseEnumMessage message2 = SparseEnumMessage.parseFrom(rawBytes); + assertEquals(TestSparseEnum.SPARSE_E, message2.getSparseEnum()); } } diff --git a/java/src/test/java/com/google/protobuf/DescriptorsTest.java b/java/src/test/java/com/google/protobuf/DescriptorsTest.java index 65d06e32..9c310919 100644 --- a/java/src/test/java/com/google/protobuf/DescriptorsTest.java +++ b/java/src/test/java/com/google/protobuf/DescriptorsTest.java @@ -31,6 +31,8 @@ package com.google.protobuf; import com.google.protobuf.DescriptorProtos.DescriptorProto; +import com.google.protobuf.DescriptorProtos.EnumDescriptorProto; +import com.google.protobuf.DescriptorProtos.EnumValueDescriptorProto; import com.google.protobuf.DescriptorProtos.FieldDescriptorProto; import com.google.protobuf.DescriptorProtos.FileDescriptorProto; import com.google.protobuf.Descriptors.DescriptorValidationException; @@ -60,6 +62,7 @@ import junit.framework.TestCase; import java.util.Arrays; import java.util.Collections; +import java.util.List; /** * Unit test for {@link Descriptors}. @@ -426,7 +429,7 @@ public class DescriptorsTest extends TestCase { UnittestEnormousDescriptor.getDescriptor() .toProto().getSerializedSize() > 65536); } - + /** * Tests that the DescriptorValidationException works as intended. */ @@ -445,7 +448,7 @@ public class DescriptorsTest extends TestCase { .build()) .build(); try { - Descriptors.FileDescriptor.buildFrom(fileDescriptorProto, + Descriptors.FileDescriptor.buildFrom(fileDescriptorProto, new FileDescriptor[0]); fail("DescriptorValidationException expected"); } catch (DescriptorValidationException e) { @@ -457,4 +460,189 @@ public class DescriptorsTest extends TestCase { assertTrue(e.getCause().getMessage().indexOf("invalid") != -1); } } + + /** + * Tests the translate/crosslink for an example where a message field's name + * and type name are the same. + */ + public void testDescriptorComplexCrosslink() throws Exception { + FileDescriptorProto fileDescriptorProto = FileDescriptorProto.newBuilder() + .setName("foo.proto") + .addMessageType(DescriptorProto.newBuilder() + .setName("Foo") + .addField(FieldDescriptorProto.newBuilder() + .setLabel(FieldDescriptorProto.Label.LABEL_OPTIONAL) + .setType(FieldDescriptorProto.Type.TYPE_INT32) + .setName("foo") + .setNumber(1) + .build()) + .build()) + .addMessageType(DescriptorProto.newBuilder() + .setName("Bar") + .addField(FieldDescriptorProto.newBuilder() + .setLabel(FieldDescriptorProto.Label.LABEL_OPTIONAL) + .setTypeName("Foo") + .setName("Foo") + .setNumber(1) + .build()) + .build()) + .build(); + // translate and crosslink + FileDescriptor file = + Descriptors.FileDescriptor.buildFrom(fileDescriptorProto, + new FileDescriptor[0]); + // verify resulting descriptors + assertNotNull(file); + List msglist = file.getMessageTypes(); + assertNotNull(msglist); + assertTrue(msglist.size() == 2); + boolean barFound = false; + for (Descriptor desc : msglist) { + if (desc.getName().equals("Bar")) { + barFound = true; + assertNotNull(desc.getFields()); + List fieldlist = desc.getFields(); + assertNotNull(fieldlist); + assertTrue(fieldlist.size() == 1); + assertTrue(fieldlist.get(0).getType() == FieldDescriptor.Type.MESSAGE); + assertTrue(fieldlist.get(0).getMessageType().getName().equals("Foo")); + } + } + assertTrue(barFound); + } + + public void testInvalidPublicDependency() throws Exception { + FileDescriptorProto fooProto = FileDescriptorProto.newBuilder() + .setName("foo.proto") .build(); + FileDescriptorProto barProto = FileDescriptorProto.newBuilder() + .setName("boo.proto") + .addDependency("foo.proto") + .addPublicDependency(1) // Error, should be 0. + .build(); + FileDescriptor fooFile = Descriptors.FileDescriptor.buildFrom(fooProto, + new FileDescriptor[0]); + try { + Descriptors.FileDescriptor.buildFrom(barProto, + new FileDescriptor[] {fooFile}); + fail("DescriptorValidationException expected"); + } catch (DescriptorValidationException e) { + assertTrue( + e.getMessage().indexOf("Invalid public dependency index.") != -1); + } + } + + public void testHiddenDependency() throws Exception { + FileDescriptorProto barProto = FileDescriptorProto.newBuilder() + .setName("bar.proto") + .addMessageType(DescriptorProto.newBuilder().setName("Bar")) + .build(); + FileDescriptorProto forwardProto = FileDescriptorProto.newBuilder() + .setName("forward.proto") + .addDependency("bar.proto") + .build(); + FileDescriptorProto fooProto = FileDescriptorProto.newBuilder() + .setName("foo.proto") + .addDependency("forward.proto") + .addMessageType(DescriptorProto.newBuilder() + .setName("Foo") + .addField(FieldDescriptorProto.newBuilder() + .setLabel(FieldDescriptorProto.Label.LABEL_OPTIONAL) + .setTypeName("Bar") + .setName("bar") + .setNumber(1))) + .build(); + FileDescriptor barFile = Descriptors.FileDescriptor.buildFrom( + barProto, new FileDescriptor[0]); + FileDescriptor forwardFile = Descriptors.FileDescriptor.buildFrom( + forwardProto, new FileDescriptor[] {barFile}); + + try { + Descriptors.FileDescriptor.buildFrom( + fooProto, new FileDescriptor[] {forwardFile}); + fail("DescriptorValidationException expected"); + } catch (DescriptorValidationException e) { + assertTrue(e.getMessage().indexOf("Bar") != -1); + assertTrue(e.getMessage().indexOf("is not defined") != -1); + } + } + + public void testPublicDependency() throws Exception { + FileDescriptorProto barProto = FileDescriptorProto.newBuilder() + .setName("bar.proto") + .addMessageType(DescriptorProto.newBuilder().setName("Bar")) + .build(); + FileDescriptorProto forwardProto = FileDescriptorProto.newBuilder() + .setName("forward.proto") + .addDependency("bar.proto") + .addPublicDependency(0) + .build(); + FileDescriptorProto fooProto = FileDescriptorProto.newBuilder() + .setName("foo.proto") + .addDependency("forward.proto") + .addMessageType(DescriptorProto.newBuilder() + .setName("Foo") + .addField(FieldDescriptorProto.newBuilder() + .setLabel(FieldDescriptorProto.Label.LABEL_OPTIONAL) + .setTypeName("Bar") + .setName("bar") + .setNumber(1))) + .build(); + FileDescriptor barFile = Descriptors.FileDescriptor.buildFrom( + barProto, new FileDescriptor[0]); + FileDescriptor forwardFile = Descriptors.FileDescriptor.buildFrom( + forwardProto, new FileDescriptor[]{barFile}); + Descriptors.FileDescriptor.buildFrom( + fooProto, new FileDescriptor[] {forwardFile}); + } + + /** + * Tests the translate/crosslink for an example with a more complex namespace + * referencing. + */ + public void testComplexNamespacePublicDependency() throws Exception { + FileDescriptorProto fooProto = FileDescriptorProto.newBuilder() + .setName("bar.proto") + .setPackage("a.b.c.d.bar.shared") + .addEnumType(EnumDescriptorProto.newBuilder() + .setName("MyEnum") + .addValue(EnumValueDescriptorProto.newBuilder() + .setName("BLAH") + .setNumber(1))) + .build(); + FileDescriptorProto barProto = FileDescriptorProto.newBuilder() + .setName("foo.proto") + .addDependency("bar.proto") + .setPackage("a.b.c.d.foo.shared") + .addMessageType(DescriptorProto.newBuilder() + .setName("MyMessage") + .addField(FieldDescriptorProto.newBuilder() + .setLabel(FieldDescriptorProto.Label.LABEL_REPEATED) + .setTypeName("bar.shared.MyEnum") + .setName("MyField") + .setNumber(1))) + .build(); + // translate and crosslink + FileDescriptor fooFile = Descriptors.FileDescriptor.buildFrom( + fooProto, new FileDescriptor[0]); + FileDescriptor barFile = Descriptors.FileDescriptor.buildFrom( + barProto, new FileDescriptor[]{fooFile}); + // verify resulting descriptors + assertNotNull(barFile); + List msglist = barFile.getMessageTypes(); + assertNotNull(msglist); + assertTrue(msglist.size() == 1); + Descriptor desc = msglist.get(0); + if (desc.getName().equals("MyMessage")) { + assertNotNull(desc.getFields()); + List fieldlist = desc.getFields(); + assertNotNull(fieldlist); + assertTrue(fieldlist.size() == 1); + FieldDescriptor field = fieldlist.get(0); + assertTrue(field.getType() == FieldDescriptor.Type.ENUM); + assertTrue(field.getEnumType().getName().equals("MyEnum")); + assertTrue(field.getEnumType().getFile().getName().equals("bar.proto")); + assertTrue(field.getEnumType().getFile().getPackage().equals( + "a.b.c.d.bar.shared")); + } + } } diff --git a/java/src/test/java/com/google/protobuf/DynamicMessageTest.java b/java/src/test/java/com/google/protobuf/DynamicMessageTest.java index aabccda2..990e8ca6 100644 --- a/java/src/test/java/com/google/protobuf/DynamicMessageTest.java +++ b/java/src/test/java/com/google/protobuf/DynamicMessageTest.java @@ -30,8 +30,9 @@ package com.google.protobuf; -import protobuf_unittest.UnittestProto.TestAllTypes; import protobuf_unittest.UnittestProto.TestAllExtensions; +import protobuf_unittest.UnittestProto.TestAllTypes; +import protobuf_unittest.UnittestProto.TestEmptyMessage; import protobuf_unittest.UnittestProto.TestPackedTypes; import junit.framework.TestCase; @@ -61,28 +62,44 @@ public class DynamicMessageTest extends TestCase { reflectionTester.assertAllFieldsSetViaReflection(message); } - public void testDoubleBuildError() throws Exception { + public void testSettersAfterBuild() throws Exception { Message.Builder builder = DynamicMessage.newBuilder(TestAllTypes.getDescriptor()); + Message firstMessage = builder.build(); + // double build() builder.build(); - try { - builder.build(); - fail("Should have thrown exception."); - } catch (IllegalStateException e) { - // Success. - } + // clear() after build() + builder.clear(); + // setters after build() + reflectionTester.setAllFieldsViaReflection(builder); + Message message = builder.build(); + reflectionTester.assertAllFieldsSetViaReflection(message); + // repeated setters after build() + reflectionTester.modifyRepeatedFieldsViaReflection(builder); + message = builder.build(); + reflectionTester.assertRepeatedFieldsModifiedViaReflection(message); + // firstMessage shouldn't have been modified. + reflectionTester.assertClearViaReflection(firstMessage); } - public void testClearAfterBuildError() throws Exception { + public void testUnknownFields() throws Exception { Message.Builder builder = - DynamicMessage.newBuilder(TestAllTypes.getDescriptor()); - builder.build(); - try { - builder.clear(); - fail("Should have thrown exception."); - } catch (IllegalStateException e) { - // Success. - } + DynamicMessage.newBuilder(TestEmptyMessage.getDescriptor()); + builder.setUnknownFields(UnknownFieldSet.newBuilder() + .addField(1, UnknownFieldSet.Field.newBuilder().addVarint(1).build()) + .addField(2, UnknownFieldSet.Field.newBuilder().addFixed32(1).build()) + .build()); + Message message = builder.build(); + assertEquals(2, message.getUnknownFields().asMap().size()); + // clone() with unknown fields + Message.Builder newBuilder = builder.clone(); + assertEquals(2, newBuilder.getUnknownFields().asMap().size()); + // clear() with unknown fields + newBuilder.clear(); + assertTrue(newBuilder.getUnknownFields().asMap().isEmpty()); + // serialize/parse with unknown fields + newBuilder.mergeFrom(message.toByteString()); + assertEquals(2, newBuilder.getUnknownFields().asMap().size()); } public void testDynamicMessageSettersRejectNull() throws Exception { @@ -167,6 +184,23 @@ public class DynamicMessageTest extends TestCase { Message message2 = DynamicMessage.parseFrom(TestAllTypes.getDescriptor(), rawBytes); reflectionTester.assertAllFieldsSetViaReflection(message2); + + // Test Parser interface. + Message message3 = message2.getParserForType().parseFrom(rawBytes); + reflectionTester.assertAllFieldsSetViaReflection(message3); + } + + public void testDynamicMessageExtensionParsing() throws Exception { + ByteString rawBytes = TestUtil.getAllExtensionsSet().toByteString(); + Message message = DynamicMessage.parseFrom( + TestAllExtensions.getDescriptor(), rawBytes, + TestUtil.getExtensionRegistry()); + extensionsReflectionTester.assertAllFieldsSetViaReflection(message); + + // Test Parser interface. + Message message2 = message.getParserForType().parseFrom( + rawBytes, TestUtil.getExtensionRegistry()); + extensionsReflectionTester.assertAllFieldsSetViaReflection(message2); } public void testDynamicMessagePackedSerialization() throws Exception { @@ -194,6 +228,10 @@ public class DynamicMessageTest extends TestCase { Message message2 = DynamicMessage.parseFrom(TestPackedTypes.getDescriptor(), rawBytes); packedReflectionTester.assertPackedFieldsSetViaReflection(message2); + + // Test Parser interface. + Message message3 = message2.getParserForType().parseFrom(rawBytes); + packedReflectionTester.assertPackedFieldsSetViaReflection(message3); } public void testDynamicMessageCopy() throws Exception { diff --git a/java/src/test/java/com/google/protobuf/GeneratedMessageTest.java b/java/src/test/java/com/google/protobuf/GeneratedMessageTest.java index acb22355..bf9db75b 100644 --- a/java/src/test/java/com/google/protobuf/GeneratedMessageTest.java +++ b/java/src/test/java/com/google/protobuf/GeneratedMessageTest.java @@ -30,6 +30,8 @@ package com.google.protobuf; +import com.google.protobuf.Descriptors.Descriptor; +import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.UnittestLite.TestAllExtensionsLite; import com.google.protobuf.test.UnittestImport; import protobuf_unittest.EnumWithNoOuter; @@ -53,6 +55,7 @@ import protobuf_unittest.UnittestProto.ForeignMessage; import protobuf_unittest.UnittestProto.ForeignMessageOrBuilder; import protobuf_unittest.UnittestProto.TestAllExtensions; import protobuf_unittest.UnittestProto.TestAllTypes; +import protobuf_unittest.UnittestProto.TestAllTypes.NestedMessage; import protobuf_unittest.UnittestProto.TestAllTypesOrBuilder; import protobuf_unittest.UnittestProto.TestExtremeDefaultValues; import protobuf_unittest.UnittestProto.TestPackedTypes; @@ -180,6 +183,33 @@ public class GeneratedMessageTest extends TestCase { assertIsUnmodifiable(value.getRepeatedFloatList()); } + public void testParsedMessagesAreImmutable() throws Exception { + TestAllTypes value = TestAllTypes.PARSER.parseFrom( + TestUtil.getAllSet().toByteString()); + assertIsUnmodifiable(value.getRepeatedInt32List()); + assertIsUnmodifiable(value.getRepeatedInt64List()); + assertIsUnmodifiable(value.getRepeatedUint32List()); + assertIsUnmodifiable(value.getRepeatedUint64List()); + assertIsUnmodifiable(value.getRepeatedSint32List()); + assertIsUnmodifiable(value.getRepeatedSint64List()); + assertIsUnmodifiable(value.getRepeatedFixed32List()); + assertIsUnmodifiable(value.getRepeatedFixed64List()); + assertIsUnmodifiable(value.getRepeatedSfixed32List()); + assertIsUnmodifiable(value.getRepeatedSfixed64List()); + assertIsUnmodifiable(value.getRepeatedFloatList()); + assertIsUnmodifiable(value.getRepeatedDoubleList()); + assertIsUnmodifiable(value.getRepeatedBoolList()); + assertIsUnmodifiable(value.getRepeatedStringList()); + assertIsUnmodifiable(value.getRepeatedBytesList()); + assertIsUnmodifiable(value.getRepeatedGroupList()); + assertIsUnmodifiable(value.getRepeatedNestedMessageList()); + assertIsUnmodifiable(value.getRepeatedForeignMessageList()); + assertIsUnmodifiable(value.getRepeatedImportMessageList()); + assertIsUnmodifiable(value.getRepeatedNestedEnumList()); + assertIsUnmodifiable(value.getRepeatedForeignEnumList()); + assertIsUnmodifiable(value.getRepeatedImportEnumList()); + } + private void assertIsUnmodifiable(List list) { if (list == Collections.emptyList()) { // OKAY -- Need to check this b/c EmptyList allows you to call clear. @@ -881,7 +911,7 @@ public class GeneratedMessageTest extends TestCase { builder.setOptionalNestedMessage(nestedMessage1); assertEquals(3, mockParent.getInvalidationCount()); - // primitive repated + // primitive repeated builder.buildPartial(); builder.addRepeatedInt32(2); builder.addRepeatedInt32(3); @@ -977,4 +1007,140 @@ public class GeneratedMessageTest extends TestCase { assertSame(b1, messageOrBuilderList.get(1)); assertSame(m2, messageOrBuilderList.get(2)); } + + public void testGetFieldBuilder() { + Descriptor descriptor = TestAllTypes.getDescriptor(); + + FieldDescriptor fieldDescriptor = + descriptor.findFieldByName("optional_nested_message"); + FieldDescriptor foreignFieldDescriptor = + descriptor.findFieldByName("optional_foreign_message"); + FieldDescriptor importFieldDescriptor = + descriptor.findFieldByName("optional_import_message"); + + // Mutate the message with new field builder + // Mutate nested message + TestAllTypes.Builder builder1 = TestAllTypes.newBuilder(); + Message.Builder fieldBuilder1 = builder1.newBuilderForField(fieldDescriptor) + .mergeFrom((Message) builder1.getField(fieldDescriptor)); + FieldDescriptor subFieldDescriptor1 = + fieldBuilder1.getDescriptorForType().findFieldByName("bb"); + fieldBuilder1.setField(subFieldDescriptor1, 1); + builder1.setField(fieldDescriptor, fieldBuilder1.build()); + + // Mutate foreign message + Message.Builder foreignFieldBuilder1 = builder1.newBuilderForField( + foreignFieldDescriptor) + .mergeFrom((Message) builder1.getField(foreignFieldDescriptor)); + FieldDescriptor subForeignFieldDescriptor1 = + foreignFieldBuilder1.getDescriptorForType().findFieldByName("c"); + foreignFieldBuilder1.setField(subForeignFieldDescriptor1, 2); + builder1.setField(foreignFieldDescriptor, foreignFieldBuilder1.build()); + + // Mutate import message + Message.Builder importFieldBuilder1 = builder1.newBuilderForField( + importFieldDescriptor) + .mergeFrom((Message) builder1.getField(importFieldDescriptor)); + FieldDescriptor subImportFieldDescriptor1 = + importFieldBuilder1.getDescriptorForType().findFieldByName("d"); + importFieldBuilder1.setField(subImportFieldDescriptor1, 3); + builder1.setField(importFieldDescriptor, importFieldBuilder1.build()); + + Message newMessage1 = builder1.build(); + + // Mutate the message with existing field builder + // Mutate nested message + TestAllTypes.Builder builder2 = TestAllTypes.newBuilder(); + Message.Builder fieldBuilder2 = builder2.getFieldBuilder(fieldDescriptor); + FieldDescriptor subFieldDescriptor2 = + fieldBuilder2.getDescriptorForType().findFieldByName("bb"); + fieldBuilder2.setField(subFieldDescriptor2, 1); + builder2.setField(fieldDescriptor, fieldBuilder2.build()); + + // Mutate foreign message + Message.Builder foreignFieldBuilder2 = builder2.newBuilderForField( + foreignFieldDescriptor) + .mergeFrom((Message) builder2.getField(foreignFieldDescriptor)); + FieldDescriptor subForeignFieldDescriptor2 = + foreignFieldBuilder2.getDescriptorForType().findFieldByName("c"); + foreignFieldBuilder2.setField(subForeignFieldDescriptor2, 2); + builder2.setField(foreignFieldDescriptor, foreignFieldBuilder2.build()); + + // Mutate import message + Message.Builder importFieldBuilder2 = builder2.newBuilderForField( + importFieldDescriptor) + .mergeFrom((Message) builder2.getField(importFieldDescriptor)); + FieldDescriptor subImportFieldDescriptor2 = + importFieldBuilder2.getDescriptorForType().findFieldByName("d"); + importFieldBuilder2.setField(subImportFieldDescriptor2, 3); + builder2.setField(importFieldDescriptor, importFieldBuilder2.build()); + + Message newMessage2 = builder2.build(); + + // These two messages should be equal. + assertEquals(newMessage1, newMessage2); + } + + public void testGetFieldBuilderWithInitializedValue() { + Descriptor descriptor = TestAllTypes.getDescriptor(); + FieldDescriptor fieldDescriptor = + descriptor.findFieldByName("optional_nested_message"); + + // Before setting field, builder is initialized by default value. + TestAllTypes.Builder builder = TestAllTypes.newBuilder(); + NestedMessage.Builder fieldBuilder = + (NestedMessage.Builder) builder.getFieldBuilder(fieldDescriptor); + assertEquals(0, fieldBuilder.getBb()); + + // Setting field value with new field builder instance. + builder = TestAllTypes.newBuilder(); + NestedMessage.Builder newFieldBuilder = + builder.getOptionalNestedMessageBuilder(); + newFieldBuilder.setBb(2); + // Then get the field builder instance by getFieldBuilder(). + fieldBuilder = + (NestedMessage.Builder) builder.getFieldBuilder(fieldDescriptor); + // It should contain new value. + assertEquals(2, fieldBuilder.getBb()); + // These two builder should be equal. + assertSame(fieldBuilder, newFieldBuilder); + } + + public void testGetFieldBuilderNotSupportedException() { + Descriptor descriptor = TestAllTypes.getDescriptor(); + TestAllTypes.Builder builder = TestAllTypes.newBuilder(); + try { + builder.getFieldBuilder(descriptor.findFieldByName("optional_int32")); + fail("Exception was not thrown"); + } catch (UnsupportedOperationException e) { + // We expect this exception. + } + try { + builder.getFieldBuilder( + descriptor.findFieldByName("optional_nested_enum")); + fail("Exception was not thrown"); + } catch (UnsupportedOperationException e) { + // We expect this exception. + } + try { + builder.getFieldBuilder(descriptor.findFieldByName("repeated_int32")); + fail("Exception was not thrown"); + } catch (UnsupportedOperationException e) { + // We expect this exception. + } + try { + builder.getFieldBuilder( + descriptor.findFieldByName("repeated_nested_enum")); + fail("Exception was not thrown"); + } catch (UnsupportedOperationException e) { + // We expect this exception. + } + try { + builder.getFieldBuilder( + descriptor.findFieldByName("repeated_nested_message")); + fail("Exception was not thrown"); + } catch (UnsupportedOperationException e) { + // We expect this exception. + } + } } diff --git a/java/src/test/java/com/google/protobuf/IsValidUtf8Test.java b/java/src/test/java/com/google/protobuf/IsValidUtf8Test.java new file mode 100644 index 00000000..b204b604 --- /dev/null +++ b/java/src/test/java/com/google/protobuf/IsValidUtf8Test.java @@ -0,0 +1,180 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// http://code.google.com/p/protobuf/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import com.google.protobuf.IsValidUtf8TestUtil.Shard; + +import junit.framework.TestCase; + +import java.io.UnsupportedEncodingException; + +/** + * Tests cases for {@link ByteString#isValidUtf8()}. This includes three + * brute force tests that actually test every permutation of one byte, two byte, + * and three byte sequences to ensure that the method produces the right result + * for every possible byte encoding where "right" means it's consistent with + * java's UTF-8 string encoding/decoding such that the method returns true for + * any sequence that will round trip when converted to a String and then back to + * bytes and will return false for any sequence that will not round trip. + * See also {@link IsValidUtf8FourByteTest}. It also includes some + * other more targeted tests. + * + * @author jonp@google.com (Jon Perlow) + * @author martinrb@google.com (Martin Buchholz) + */ +public class IsValidUtf8Test extends TestCase { + + /** + * Tests that round tripping of all two byte permutations work. + */ + public void testIsValidUtf8_1Byte() throws UnsupportedEncodingException { + IsValidUtf8TestUtil.testBytes(1, + IsValidUtf8TestUtil.EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT); + } + + /** + * Tests that round tripping of all two byte permutations work. + */ + public void testIsValidUtf8_2Bytes() throws UnsupportedEncodingException { + IsValidUtf8TestUtil.testBytes(2, + IsValidUtf8TestUtil.EXPECTED_TWO_BYTE_ROUNDTRIPPABLE_COUNT); + } + + /** + * Tests that round tripping of all three byte permutations work. + */ + public void testIsValidUtf8_3Bytes() throws UnsupportedEncodingException { + IsValidUtf8TestUtil.testBytes(3, + IsValidUtf8TestUtil.EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT); + } + + /** + * Tests that round tripping of a sample of four byte permutations work. + * All permutations are prohibitively expensive to test for automated runs; + * {@link IsValidUtf8FourByteTest} is used for full coverage. This method + * tests specific four-byte cases. + */ + public void testIsValidUtf8_4BytesSamples() + throws UnsupportedEncodingException { + // Valid 4 byte. + assertValidUtf8(0xF0, 0xA4, 0xAD, 0xA2); + + // Bad trailing bytes + assertInvalidUtf8(0xF0, 0xA4, 0xAD, 0x7F); + assertInvalidUtf8(0xF0, 0xA4, 0xAD, 0xC0); + + // Special cases for byte2 + assertInvalidUtf8(0xF0, 0x8F, 0xAD, 0xA2); + assertInvalidUtf8(0xF4, 0x90, 0xAD, 0xA2); + } + + /** + * Tests some hard-coded test cases. + */ + public void testSomeSequences() { + // Empty + assertTrue(asBytes("").isValidUtf8()); + + // One-byte characters, including control characters + assertTrue(asBytes("\u0000abc\u007f").isValidUtf8()); + + // Two-byte characters + assertTrue(asBytes("\u00a2\u00a2").isValidUtf8()); + + // Three-byte characters + assertTrue(asBytes("\u020ac\u020ac").isValidUtf8()); + + // Four-byte characters + assertTrue(asBytes("\u024B62\u024B62").isValidUtf8()); + + // Mixed string + assertTrue( + asBytes("a\u020ac\u00a2b\\u024B62u020acc\u00a2de\u024B62") + .isValidUtf8()); + + // Not a valid string + assertInvalidUtf8(-1, 0, -1, 0); + } + + private byte[] toByteArray(int... bytes) { + byte[] realBytes = new byte[bytes.length]; + for (int i = 0; i < bytes.length; i++) { + realBytes[i] = (byte) bytes[i]; + } + return realBytes; + } + + private ByteString toByteString(int... bytes) { + return ByteString.copyFrom(toByteArray(bytes)); + } + + private void assertValidUtf8(int[] bytes, boolean not) { + byte[] realBytes = toByteArray(bytes); + assertTrue(not ^ Utf8.isValidUtf8(realBytes)); + assertTrue(not ^ Utf8.isValidUtf8(realBytes, 0, bytes.length)); + ByteString lit = ByteString.copyFrom(realBytes); + ByteString sub = lit.substring(0, bytes.length); + assertTrue(not ^ lit.isValidUtf8()); + assertTrue(not ^ sub.isValidUtf8()); + ByteString[] ropes = { + RopeByteString.newInstanceForTest(ByteString.EMPTY, lit), + RopeByteString.newInstanceForTest(ByteString.EMPTY, sub), + RopeByteString.newInstanceForTest(lit, ByteString.EMPTY), + RopeByteString.newInstanceForTest(sub, ByteString.EMPTY), + RopeByteString.newInstanceForTest(sub, lit) + }; + for (ByteString rope : ropes) { + assertTrue(not ^ rope.isValidUtf8()); + } + } + + private void assertValidUtf8(int... bytes) { + assertValidUtf8(bytes, false); + } + + private void assertInvalidUtf8(int... bytes) { + assertValidUtf8(bytes, true); + } + + private static ByteString asBytes(String s) { + return ByteString.copyFromUtf8(s); + } + + public void testShardsHaveExpectedRoundTrippables() { + // A sanity check. + int actual = 0; + for (Shard shard : IsValidUtf8TestUtil.FOUR_BYTE_SHARDS) { + actual += shard.expected; + } + assertEquals(IsValidUtf8TestUtil.EXPECTED_FOUR_BYTE_ROUNDTRIPPABLE_COUNT, + actual); + } +} diff --git a/java/src/test/java/com/google/protobuf/IsValidUtf8TestUtil.java b/java/src/test/java/com/google/protobuf/IsValidUtf8TestUtil.java new file mode 100644 index 00000000..4cb3d5b9 --- /dev/null +++ b/java/src/test/java/com/google/protobuf/IsValidUtf8TestUtil.java @@ -0,0 +1,421 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// http://code.google.com/p/protobuf/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import static junit.framework.Assert.*; + +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.logging.Logger; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.Charset; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CoderResult; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; + +/** + * Shared testing code for {@link IsValidUtf8Test} and + * {@link IsValidUtf8FourByteTest}. + * + * @author jonp@google.com (Jon Perlow) + * @author martinrb@google.com (Martin Buchholz) + */ +class IsValidUtf8TestUtil { + private static Logger logger = Logger.getLogger( + IsValidUtf8TestUtil.class.getName()); + + // 128 - [chars 0x0000 to 0x007f] + static long ONE_BYTE_ROUNDTRIPPABLE_CHARACTERS = 0x007f - 0x0000 + 1; + + // 128 + static long EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT = + ONE_BYTE_ROUNDTRIPPABLE_CHARACTERS; + + // 1920 [chars 0x0080 to 0x07FF] + static long TWO_BYTE_ROUNDTRIPPABLE_CHARACTERS = 0x07FF - 0x0080 + 1; + + // 18,304 + static long EXPECTED_TWO_BYTE_ROUNDTRIPPABLE_COUNT = + // Both bytes are one byte characters + (long) Math.pow(EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT, 2) + + // The possible number of two byte characters + TWO_BYTE_ROUNDTRIPPABLE_CHARACTERS; + + // 2048 + static long THREE_BYTE_SURROGATES = 2 * 1024; + + // 61,440 [chars 0x0800 to 0xFFFF, minus surrogates] + static long THREE_BYTE_ROUNDTRIPPABLE_CHARACTERS = + 0xFFFF - 0x0800 + 1 - THREE_BYTE_SURROGATES; + + // 2,650,112 + static long EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT = + // All one byte characters + (long) Math.pow(EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT, 3) + + // One two byte character and a one byte character + 2 * TWO_BYTE_ROUNDTRIPPABLE_CHARACTERS * + ONE_BYTE_ROUNDTRIPPABLE_CHARACTERS + + // Three byte characters + THREE_BYTE_ROUNDTRIPPABLE_CHARACTERS; + + // 1,048,576 [chars 0x10000L to 0x10FFFF] + static long FOUR_BYTE_ROUNDTRIPPABLE_CHARACTERS = 0x10FFFF - 0x10000L + 1; + + // 289,571,839 + static long EXPECTED_FOUR_BYTE_ROUNDTRIPPABLE_COUNT = + // All one byte characters + (long) Math.pow(EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT, 4) + + // One and three byte characters + 2 * THREE_BYTE_ROUNDTRIPPABLE_CHARACTERS * + ONE_BYTE_ROUNDTRIPPABLE_CHARACTERS + + // Two two byte characters + TWO_BYTE_ROUNDTRIPPABLE_CHARACTERS * TWO_BYTE_ROUNDTRIPPABLE_CHARACTERS + + // Permutations of one and two byte characters + 3 * TWO_BYTE_ROUNDTRIPPABLE_CHARACTERS * + ONE_BYTE_ROUNDTRIPPABLE_CHARACTERS * + ONE_BYTE_ROUNDTRIPPABLE_CHARACTERS + + // Four byte characters + FOUR_BYTE_ROUNDTRIPPABLE_CHARACTERS; + + static class Shard { + final long index; + final long start; + final long lim; + final long expected; + + + public Shard(long index, long start, long lim, long expected) { + assertTrue(start < lim); + this.index = index; + this.start = start; + this.lim = lim; + this.expected = expected; + } + } + + static final long[] FOUR_BYTE_SHARDS_EXPECTED_ROUNTRIPPABLES = + generateFourByteShardsExpectedRunnables(); + + private static long[] generateFourByteShardsExpectedRunnables() { + long[] expected = new long[128]; + + // 0-63 are all 5300224 + for (int i = 0; i <= 63; i++) { + expected[i] = 5300224; + } + + // 97-111 are all 2342912 + for (int i = 97; i <= 111; i++) { + expected[i] = 2342912; + } + + // 113-117 are all 1048576 + for (int i = 113; i <= 117; i++) { + expected[i] = 1048576; + } + + // One offs + expected[112] = 786432; + expected[118] = 786432; + expected[119] = 1048576; + expected[120] = 458752; + expected[121] = 524288; + expected[122] = 65536; + + // Anything not assigned was the default 0. + return expected; + } + + static final List FOUR_BYTE_SHARDS = generateFourByteShards( + 128, FOUR_BYTE_SHARDS_EXPECTED_ROUNTRIPPABLES); + + + private static List generateFourByteShards( + int numShards, long[] expected) { + assertEquals(numShards, expected.length); + List shards = new ArrayList(numShards); + long LIM = 1L << 32; + long increment = LIM / numShards; + assertTrue(LIM % numShards == 0); + for (int i = 0; i < numShards; i++) { + shards.add(new Shard(i, + increment * i, + increment * (i + 1), + expected[i])); + } + return shards; + } + + /** + * Helper to run the loop to test all the permutations for the number of bytes + * specified. + * + * @param numBytes the number of bytes in the byte array + * @param expectedCount the expected number of roundtrippable permutations + */ + static void testBytes(int numBytes, long expectedCount) + throws UnsupportedEncodingException { + testBytes(numBytes, expectedCount, 0, -1); + } + + /** + * Helper to run the loop to test all the permutations for the number of bytes + * specified. This overload is useful for debugging to get the loop to start + * at a certain character. + * + * @param numBytes the number of bytes in the byte array + * @param expectedCount the expected number of roundtrippable permutations + * @param start the starting bytes encoded as a long as big-endian + * @param lim the limit of bytes to process encoded as a long as big-endian, + * or -1 to mean the max limit for numBytes + */ + static void testBytes(int numBytes, long expectedCount, long start, long lim) + throws UnsupportedEncodingException { + Random rnd = new Random(); + byte[] bytes = new byte[numBytes]; + + if (lim == -1) { + lim = 1L << (numBytes * 8); + } + long count = 0; + long countRoundTripped = 0; + for (long byteChar = start; byteChar < lim; byteChar++) { + long tmpByteChar = byteChar; + for (int i = 0; i < numBytes; i++) { + bytes[bytes.length - i - 1] = (byte) tmpByteChar; + tmpByteChar = tmpByteChar >> 8; + } + ByteString bs = ByteString.copyFrom(bytes); + boolean isRoundTrippable = bs.isValidUtf8(); + String s = new String(bytes, "UTF-8"); + byte[] bytesReencoded = s.getBytes("UTF-8"); + boolean bytesEqual = Arrays.equals(bytes, bytesReencoded); + + if (bytesEqual != isRoundTrippable) { + outputFailure(byteChar, bytes, bytesReencoded); + } + + // Check agreement with static Utf8 methods. + assertEquals(isRoundTrippable, Utf8.isValidUtf8(bytes)); + assertEquals(isRoundTrippable, Utf8.isValidUtf8(bytes, 0, numBytes)); + + // Test partial sequences. + // Partition numBytes into three segments (not necessarily non-empty). + int i = rnd.nextInt(numBytes); + int j = rnd.nextInt(numBytes); + if (j < i) { + int tmp = i; i = j; j = tmp; + } + int state1 = Utf8.partialIsValidUtf8(Utf8.COMPLETE, bytes, 0, i); + int state2 = Utf8.partialIsValidUtf8(state1, bytes, i, j); + int state3 = Utf8.partialIsValidUtf8(state2, bytes, j, numBytes); + if (isRoundTrippable != (state3 == Utf8.COMPLETE)) { + System.out.printf("state=%04x %04x %04x i=%d j=%d%n", + state1, state2, state3, i, j); + outputFailure(byteChar, bytes, bytesReencoded); + } + assertEquals(isRoundTrippable, (state3 == Utf8.COMPLETE)); + + // Test ropes built out of small partial sequences + ByteString rope = RopeByteString.newInstanceForTest( + bs.substring(0, i), + RopeByteString.newInstanceForTest( + bs.substring(i, j), + bs.substring(j, numBytes))); + assertSame(RopeByteString.class, rope.getClass()); + + ByteString[] byteStrings = { bs, bs.substring(0, numBytes), rope }; + for (ByteString x : byteStrings) { + assertEquals(isRoundTrippable, + x.isValidUtf8()); + assertEquals(state3, + x.partialIsValidUtf8(Utf8.COMPLETE, 0, numBytes)); + + assertEquals(state1, + x.partialIsValidUtf8(Utf8.COMPLETE, 0, i)); + assertEquals(state1, + x.substring(0, i).partialIsValidUtf8(Utf8.COMPLETE, 0, i)); + assertEquals(state2, + x.partialIsValidUtf8(state1, i, j - i)); + assertEquals(state2, + x.substring(i, j).partialIsValidUtf8(state1, 0, j - i)); + assertEquals(state3, + x.partialIsValidUtf8(state2, j, numBytes - j)); + assertEquals(state3, + x.substring(j, numBytes) + .partialIsValidUtf8(state2, 0, numBytes - j)); + } + + // ByteString reduplication should not affect its UTF-8 validity. + ByteString ropeADope = + RopeByteString.newInstanceForTest(bs, bs.substring(0, numBytes)); + assertEquals(isRoundTrippable, ropeADope.isValidUtf8()); + + if (isRoundTrippable) { + countRoundTripped++; + } + count++; + if (byteChar != 0 && byteChar % 1000000L == 0) { + logger.info("Processed " + (byteChar / 1000000L) + + " million characters"); + } + } + logger.info("Round tripped " + countRoundTripped + " of " + count); + assertEquals(expectedCount, countRoundTripped); + } + + /** + * Variation of {@link #testBytes} that does less allocation using the + * low-level encoders/decoders directly. Checked in because it's useful for + * debugging when trying to process bytes faster, but since it doesn't use the + * actual String class, it's possible for incompatibilities to develop + * (although unlikely). + * + * @param numBytes the number of bytes in the byte array + * @param expectedCount the expected number of roundtrippable permutations + * @param start the starting bytes encoded as a long as big-endian + * @param lim the limit of bytes to process encoded as a long as big-endian, + * or -1 to mean the max limit for numBytes + */ + void testBytesUsingByteBuffers( + int numBytes, long expectedCount, long start, long lim) + throws UnsupportedEncodingException { + CharsetDecoder decoder = Charset.forName("UTF-8").newDecoder() + .onMalformedInput(CodingErrorAction.REPLACE) + .onUnmappableCharacter(CodingErrorAction.REPLACE); + CharsetEncoder encoder = Charset.forName("UTF-8").newEncoder() + .onMalformedInput(CodingErrorAction.REPLACE) + .onUnmappableCharacter(CodingErrorAction.REPLACE); + byte[] bytes = new byte[numBytes]; + int maxChars = (int) (decoder.maxCharsPerByte() * numBytes) + 1; + char[] charsDecoded = + new char[(int) (decoder.maxCharsPerByte() * numBytes) + 1]; + int maxBytes = (int) (encoder.maxBytesPerChar() * maxChars) + 1; + byte[] bytesReencoded = new byte[maxBytes]; + + ByteBuffer bb = ByteBuffer.wrap(bytes); + CharBuffer cb = CharBuffer.wrap(charsDecoded); + ByteBuffer bbReencoded = ByteBuffer.wrap(bytesReencoded); + if (lim == -1) { + lim = 1L << (numBytes * 8); + } + long count = 0; + long countRoundTripped = 0; + for (long byteChar = start; byteChar < lim; byteChar++) { + bb.rewind(); + bb.limit(bytes.length); + cb.rewind(); + cb.limit(charsDecoded.length); + bbReencoded.rewind(); + bbReencoded.limit(bytesReencoded.length); + encoder.reset(); + decoder.reset(); + long tmpByteChar = byteChar; + for (int i = 0; i < bytes.length; i++) { + bytes[bytes.length - i - 1] = (byte) tmpByteChar; + tmpByteChar = tmpByteChar >> 8; + } + boolean isRoundTrippable = ByteString.copyFrom(bytes).isValidUtf8(); + CoderResult result = decoder.decode(bb, cb, true); + assertFalse(result.isError()); + result = decoder.flush(cb); + assertFalse(result.isError()); + + int charLen = cb.position(); + cb.rewind(); + cb.limit(charLen); + result = encoder.encode(cb, bbReencoded, true); + assertFalse(result.isError()); + result = encoder.flush(bbReencoded); + assertFalse(result.isError()); + + boolean bytesEqual = true; + int bytesLen = bbReencoded.position(); + if (bytesLen != numBytes) { + bytesEqual = false; + } else { + for (int i = 0; i < numBytes; i++) { + if (bytes[i] != bytesReencoded[i]) { + bytesEqual = false; + break; + } + } + } + if (bytesEqual != isRoundTrippable) { + outputFailure(byteChar, bytes, bytesReencoded, bytesLen); + } + + count++; + if (isRoundTrippable) { + countRoundTripped++; + } + if (byteChar != 0 && byteChar % 1000000 == 0) { + logger.info("Processed " + (byteChar / 1000000) + + " million characters"); + } + } + logger.info("Round tripped " + countRoundTripped + " of " + count); + assertEquals(expectedCount, countRoundTripped); + } + + private static void outputFailure(long byteChar, byte[] bytes, byte[] after) { + outputFailure(byteChar, bytes, after, after.length); + } + + private static void outputFailure(long byteChar, byte[] bytes, byte[] after, + int len) { + fail("Failure: (" + Long.toHexString(byteChar) + ") " + + toHexString(bytes) + " => " + toHexString(after, len)); + } + + private static String toHexString(byte[] b) { + return toHexString(b, b.length); + } + + private static String toHexString(byte[] b, int len) { + StringBuilder s = new StringBuilder(); + s.append("\""); + for (int i = 0; i < len; i++) { + if (i > 0) { + s.append(" "); + } + s.append(String.format("%02x", b[i] & 0xFF)); + } + s.append("\""); + return s.toString(); + } + +} diff --git a/java/src/test/java/com/google/protobuf/LazyStringArrayListTest.java b/java/src/test/java/com/google/protobuf/LazyStringArrayListTest.java index 4dcdc74d..d500595f 100644 --- a/java/src/test/java/com/google/protobuf/LazyStringArrayListTest.java +++ b/java/src/test/java/com/google/protobuf/LazyStringArrayListTest.java @@ -32,6 +32,9 @@ package com.google.protobuf; import junit.framework.TestCase; +import java.util.ArrayList; +import java.util.List; + /** * Tests for {@link LazyStringArrayList}. * @@ -115,4 +118,45 @@ public class LazyStringArrayListTest extends TestCase { assertSame(aPrimeByteString, list.getByteString(0)); assertSame(bPrimeByteString, list.getByteString(1)); } + + public void testCopyConstructorCopiesByReference() { + LazyStringArrayList list1 = new LazyStringArrayList(); + list1.add(STRING_A); + list1.add(BYTE_STRING_B); + list1.add(BYTE_STRING_C); + + LazyStringArrayList list2 = new LazyStringArrayList(list1); + assertEquals(3, list2.size()); + assertSame(STRING_A, list2.get(0)); + assertSame(BYTE_STRING_B, list2.getByteString(1)); + assertSame(BYTE_STRING_C, list2.getByteString(2)); + } + + public void testListCopyConstructor() { + List list1 = new ArrayList(); + list1.add(STRING_A); + list1.add(STRING_B); + list1.add(STRING_C); + + LazyStringArrayList list2 = new LazyStringArrayList(list1); + assertEquals(3, list2.size()); + assertSame(STRING_A, list2.get(0)); + assertSame(STRING_B, list2.get(1)); + assertSame(STRING_C, list2.get(2)); + } + + public void testAddAllCopiesByReferenceIfPossible() { + LazyStringArrayList list1 = new LazyStringArrayList(); + list1.add(STRING_A); + list1.add(BYTE_STRING_B); + list1.add(BYTE_STRING_C); + + LazyStringArrayList list2 = new LazyStringArrayList(); + list2.addAll(list1); + + assertEquals(3, list2.size()); + assertSame(STRING_A, list2.get(0)); + assertSame(BYTE_STRING_B, list2.getByteString(1)); + assertSame(BYTE_STRING_C, list2.getByteString(2)); + } } diff --git a/java/src/test/java/com/google/protobuf/LazyStringEndToEndTest.java b/java/src/test/java/com/google/protobuf/LazyStringEndToEndTest.java index e6870b51..fe9599e3 100644 --- a/java/src/test/java/com/google/protobuf/LazyStringEndToEndTest.java +++ b/java/src/test/java/com/google/protobuf/LazyStringEndToEndTest.java @@ -50,6 +50,19 @@ public class LazyStringEndToEndTest extends TestCase { 114, 4, -1, 0, -1, 0, -30, 2, 4, -1, 0, -1, 0, -30, 2, 4, -1, 0, -1, 0, }); + private ByteString encodedTestAllTypes; + + @Override + protected void setUp() throws Exception { + super.setUp(); + this.encodedTestAllTypes = UnittestProto.TestAllTypes.newBuilder() + .setOptionalString("foo") + .addRepeatedString("bar") + .addRepeatedString("baz") + .build() + .toByteString(); + } + /** * Tests that an invalid UTF8 string will roundtrip through a parse * and serialization. @@ -112,4 +125,19 @@ public class LazyStringEndToEndTest extends TestCase { assertSame(bPrime, proto.getRepeatedString(0)); assertSame(cPrime, proto.getRepeatedString(1)); } + + public void testNoStringCachingIfOnlyBytesAccessed() throws Exception { + UnittestProto.TestAllTypes proto = + UnittestProto.TestAllTypes.parseFrom(encodedTestAllTypes); + ByteString optional = proto.getOptionalStringBytes(); + assertSame(optional, proto.getOptionalStringBytes()); + assertSame(optional, proto.toBuilder().getOptionalStringBytes()); + + ByteString repeated0 = proto.getRepeatedStringBytes(0); + ByteString repeated1 = proto.getRepeatedStringBytes(1); + assertSame(repeated0, proto.getRepeatedStringBytes(0)); + assertSame(repeated1, proto.getRepeatedStringBytes(1)); + assertSame(repeated0, proto.toBuilder().getRepeatedStringBytes(0)); + assertSame(repeated1, proto.toBuilder().getRepeatedStringBytes(1)); + } } diff --git a/java/src/test/java/com/google/protobuf/LiteralByteStringTest.java b/java/src/test/java/com/google/protobuf/LiteralByteStringTest.java new file mode 100644 index 00000000..deee1eea --- /dev/null +++ b/java/src/test/java/com/google/protobuf/LiteralByteStringTest.java @@ -0,0 +1,396 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// http://code.google.com/p/protobuf/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import junit.framework.TestCase; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; +import java.util.NoSuchElementException; + +/** + * Test {@link LiteralByteString} by setting up a reference string in {@link #setUp()}. + * This class is designed to be extended for testing extensions of {@link LiteralByteString} + * such as {@link BoundedByteString}, see {@link BoundedByteStringTest}. + * + * @author carlanton@google.com (Carl Haverl) + */ +public class LiteralByteStringTest extends TestCase { + protected static final String UTF_8 = "UTF-8"; + + protected String classUnderTest; + protected byte[] referenceBytes; + protected ByteString stringUnderTest; + protected int expectedHashCode; + + @Override + protected void setUp() throws Exception { + classUnderTest = "LiteralByteString"; + referenceBytes = ByteStringTest.getTestBytes(1234, 11337766L); + stringUnderTest = ByteString.copyFrom(referenceBytes); + expectedHashCode = 331161852; + } + + public void testExpectedType() { + String actualClassName = getActualClassName(stringUnderTest); + assertEquals(classUnderTest + " should match type exactly", classUnderTest, actualClassName); + } + + protected String getActualClassName(Object object) { + String actualClassName = object.getClass().getName(); + actualClassName = actualClassName.substring(actualClassName.lastIndexOf('.') + 1); + return actualClassName; + } + + public void testByteAt() { + boolean stillEqual = true; + for (int i = 0; stillEqual && i < referenceBytes.length; ++i) { + stillEqual = (referenceBytes[i] == stringUnderTest.byteAt(i)); + } + assertTrue(classUnderTest + " must capture the right bytes", stillEqual); + } + + public void testByteIterator() { + boolean stillEqual = true; + ByteString.ByteIterator iter = stringUnderTest.iterator(); + for (int i = 0; stillEqual && i < referenceBytes.length; ++i) { + stillEqual = (iter.hasNext() && referenceBytes[i] == iter.nextByte()); + } + assertTrue(classUnderTest + " must capture the right bytes", stillEqual); + assertFalse(classUnderTest + " must have exhausted the itertor", iter.hasNext()); + + try { + iter.nextByte(); + fail("Should have thrown an exception."); + } catch (NoSuchElementException e) { + // This is success + } + } + + public void testByteIterable() { + boolean stillEqual = true; + int j = 0; + for (byte quantum : stringUnderTest) { + stillEqual = (referenceBytes[j] == quantum); + ++j; + } + assertTrue(classUnderTest + " must capture the right bytes as Bytes", stillEqual); + assertEquals(classUnderTest + " iterable character count", referenceBytes.length, j); + } + + public void testSize() { + assertEquals(classUnderTest + " must have the expected size", referenceBytes.length, + stringUnderTest.size()); + } + + public void testGetTreeDepth() { + assertEquals(classUnderTest + " must have depth 0", 0, stringUnderTest.getTreeDepth()); + } + + public void testIsBalanced() { + assertTrue(classUnderTest + " is technically balanced", stringUnderTest.isBalanced()); + } + + public void testCopyTo_ByteArrayOffsetLength() { + int destinationOffset = 50; + int length = 100; + byte[] destination = new byte[destinationOffset + length]; + int sourceOffset = 213; + stringUnderTest.copyTo(destination, sourceOffset, destinationOffset, length); + boolean stillEqual = true; + for (int i = 0; stillEqual && i < length; ++i) { + stillEqual = referenceBytes[i + sourceOffset] == destination[i + destinationOffset]; + } + assertTrue(classUnderTest + ".copyTo(4 arg) must give the expected bytes", stillEqual); + } + + public void testCopyTo_ByteArrayOffsetLengthErrors() { + int destinationOffset = 50; + int length = 100; + byte[] destination = new byte[destinationOffset + length]; + + try { + // Copy one too many bytes + stringUnderTest.copyTo(destination, stringUnderTest.size() + 1 - length, + destinationOffset, length); + fail("Should have thrown an exception when copying too many bytes of a " + + classUnderTest); + } catch (IndexOutOfBoundsException expected) { + // This is success + } + + try { + // Copy with illegal negative sourceOffset + stringUnderTest.copyTo(destination, -1, destinationOffset, length); + fail("Should have thrown an exception when given a negative sourceOffset in " + + classUnderTest); + } catch (IndexOutOfBoundsException expected) { + // This is success + } + + try { + // Copy with illegal negative destinationOffset + stringUnderTest.copyTo(destination, 0, -1, length); + fail("Should have thrown an exception when given a negative destinationOffset in " + + classUnderTest); + } catch (IndexOutOfBoundsException expected) { + // This is success + } + + try { + // Copy with illegal negative size + stringUnderTest.copyTo(destination, 0, 0, -1); + fail("Should have thrown an exception when given a negative size in " + + classUnderTest); + } catch (IndexOutOfBoundsException expected) { + // This is success + } + + try { + // Copy with illegal too-large sourceOffset + stringUnderTest.copyTo(destination, 2 * stringUnderTest.size(), 0, length); + fail("Should have thrown an exception when the destinationOffset is too large in " + + classUnderTest); + } catch (IndexOutOfBoundsException expected) { + // This is success + } + + try { + // Copy with illegal too-large destinationOffset + stringUnderTest.copyTo(destination, 0, 2 * destination.length, length); + fail("Should have thrown an exception when the destinationOffset is too large in " + + classUnderTest); + } catch (IndexOutOfBoundsException expected) { + // This is success + } + } + + public void testCopyTo_ByteBuffer() { + ByteBuffer myBuffer = ByteBuffer.allocate(referenceBytes.length); + stringUnderTest.copyTo(myBuffer); + assertTrue(classUnderTest + ".copyTo(ByteBuffer) must give back the same bytes", + Arrays.equals(referenceBytes, myBuffer.array())); + } + + public void testAsReadOnlyByteBuffer() { + ByteBuffer byteBuffer = stringUnderTest.asReadOnlyByteBuffer(); + byte[] roundTripBytes = new byte[referenceBytes.length]; + assertTrue(byteBuffer.remaining() == referenceBytes.length); + assertTrue(byteBuffer.isReadOnly()); + byteBuffer.get(roundTripBytes); + assertTrue(classUnderTest + ".asReadOnlyByteBuffer() must give back the same bytes", + Arrays.equals(referenceBytes, roundTripBytes)); + } + + public void testAsReadOnlyByteBufferList() { + List byteBuffers = stringUnderTest.asReadOnlyByteBufferList(); + int bytesSeen = 0; + byte[] roundTripBytes = new byte[referenceBytes.length]; + for (ByteBuffer byteBuffer : byteBuffers) { + int thisLength = byteBuffer.remaining(); + assertTrue(byteBuffer.isReadOnly()); + assertTrue(bytesSeen + thisLength <= referenceBytes.length); + byteBuffer.get(roundTripBytes, bytesSeen, thisLength); + bytesSeen += thisLength; + } + assertTrue(bytesSeen == referenceBytes.length); + assertTrue(classUnderTest + ".asReadOnlyByteBufferTest() must give back the same bytes", + Arrays.equals(referenceBytes, roundTripBytes)); + } + + public void testToByteArray() { + byte[] roundTripBytes = stringUnderTest.toByteArray(); + assertTrue(classUnderTest + ".toByteArray() must give back the same bytes", + Arrays.equals(referenceBytes, roundTripBytes)); + } + + public void testWriteTo() throws IOException { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + stringUnderTest.writeTo(bos); + byte[] roundTripBytes = bos.toByteArray(); + assertTrue(classUnderTest + ".writeTo() must give back the same bytes", + Arrays.equals(referenceBytes, roundTripBytes)); + } + + public void testWriteTo_mutating() throws IOException { + OutputStream os = new OutputStream() { + @Override + public void write(byte[] b, int off, int len) { + for (int x = 0; x < len; ++x) { + b[off + x] = (byte) 0; + } + } + + @Override + public void write(int b) { + // Purposefully left blank. + } + }; + + stringUnderTest.writeTo(os); + byte[] newBytes = stringUnderTest.toByteArray(); + assertTrue(classUnderTest + ".writeTo() must not grant access to underlying array", + Arrays.equals(referenceBytes, newBytes)); + } + + public void testNewOutput() throws IOException { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ByteString.Output output = ByteString.newOutput(); + stringUnderTest.writeTo(output); + assertEquals("Output Size returns correct result", + output.size(), stringUnderTest.size()); + output.writeTo(bos); + assertTrue("Output.writeTo() must give back the same bytes", + Arrays.equals(referenceBytes, bos.toByteArray())); + + // write the output stream to itself! This should cause it to double + output.writeTo(output); + assertEquals("Writing an output stream to itself is successful", + stringUnderTest.concat(stringUnderTest), output.toByteString()); + + output.reset(); + assertEquals("Output.reset() resets the output", 0, output.size()); + assertEquals("Output.reset() resets the output", + ByteString.EMPTY, output.toByteString()); + + } + + public void testToString() throws UnsupportedEncodingException { + String testString = "I love unicode \u1234\u5678 characters"; + LiteralByteString unicode = new LiteralByteString(testString.getBytes(UTF_8)); + String roundTripString = unicode.toString(UTF_8); + assertEquals(classUnderTest + " unicode must match", testString, roundTripString); + } + + public void testEquals() { + assertEquals(classUnderTest + " must not equal null", false, stringUnderTest.equals(null)); + assertEquals(classUnderTest + " must equal self", stringUnderTest, stringUnderTest); + assertFalse(classUnderTest + " must not equal the empty string", + stringUnderTest.equals(ByteString.EMPTY)); + assertEquals(classUnderTest + " empty strings must be equal", + new LiteralByteString(new byte[]{}), stringUnderTest.substring(55, 55)); + assertEquals(classUnderTest + " must equal another string with the same value", + stringUnderTest, new LiteralByteString(referenceBytes)); + + byte[] mungedBytes = new byte[referenceBytes.length]; + System.arraycopy(referenceBytes, 0, mungedBytes, 0, referenceBytes.length); + mungedBytes[mungedBytes.length - 5] ^= 0xFF; + assertFalse(classUnderTest + " must not equal every string with the same length", + stringUnderTest.equals(new LiteralByteString(mungedBytes))); + } + + public void testHashCode() { + int hash = stringUnderTest.hashCode(); + assertEquals(classUnderTest + " must have expected hashCode", expectedHashCode, hash); + } + + public void testPeekCachedHashCode() { + assertEquals(classUnderTest + ".peekCachedHashCode() should return zero at first", 0, + stringUnderTest.peekCachedHashCode()); + stringUnderTest.hashCode(); + assertEquals(classUnderTest + ".peekCachedHashCode should return zero at first", + expectedHashCode, stringUnderTest.peekCachedHashCode()); + } + + public void testPartialHash() { + // partialHash() is more strenuously tested elsewhere by testing hashes of substrings. + // This test would fail if the expected hash were 1. It's not. + int hash = stringUnderTest.partialHash(stringUnderTest.size(), 0, stringUnderTest.size()); + assertEquals(classUnderTest + ".partialHash() must yield expected hashCode", + expectedHashCode, hash); + } + + public void testNewInput() throws IOException { + InputStream input = stringUnderTest.newInput(); + assertEquals("InputStream.available() returns correct value", + stringUnderTest.size(), input.available()); + boolean stillEqual = true; + for (byte referenceByte : referenceBytes) { + int expectedInt = (referenceByte & 0xFF); + stillEqual = (expectedInt == input.read()); + } + assertEquals("InputStream.available() returns correct value", + 0, input.available()); + assertTrue(classUnderTest + " must give the same bytes from the InputStream", stillEqual); + assertEquals(classUnderTest + " InputStream must now be exhausted", -1, input.read()); + } + + public void testNewInput_skip() throws IOException { + InputStream input = stringUnderTest.newInput(); + int stringSize = stringUnderTest.size(); + int nearEndIndex = stringSize * 2 / 3; + long skipped1 = input.skip(nearEndIndex); + assertEquals("InputStream.skip()", skipped1, nearEndIndex); + assertEquals("InputStream.available()", + stringSize - skipped1, input.available()); + assertTrue("InputStream.mark() is available", input.markSupported()); + input.mark(0); + assertEquals("InputStream.skip(), read()", + stringUnderTest.byteAt(nearEndIndex) & 0xFF, input.read()); + assertEquals("InputStream.available()", + stringSize - skipped1 - 1, input.available()); + long skipped2 = input.skip(stringSize); + assertEquals("InputStream.skip() incomplete", + skipped2, stringSize - skipped1 - 1); + assertEquals("InputStream.skip(), no more input", 0, input.available()); + assertEquals("InputStream.skip(), no more input", -1, input.read()); + input.reset(); + assertEquals("InputStream.reset() succeded", + stringSize - skipped1, input.available()); + assertEquals("InputStream.reset(), read()", + stringUnderTest.byteAt(nearEndIndex) & 0xFF, input.read()); + } + + public void testNewCodedInput() throws IOException { + CodedInputStream cis = stringUnderTest.newCodedInput(); + byte[] roundTripBytes = cis.readRawBytes(referenceBytes.length); + assertTrue(classUnderTest + " must give the same bytes back from the CodedInputStream", + Arrays.equals(referenceBytes, roundTripBytes)); + assertTrue(classUnderTest + " CodedInputStream must now be exhausted", cis.isAtEnd()); + } + + /** + * Make sure we keep things simple when concatenating with empty. See also + * {@link ByteStringTest#testConcat_empty()}. + */ + public void testConcat_empty() { + assertSame(classUnderTest + " concatenated with empty must give " + classUnderTest, + stringUnderTest.concat(ByteString.EMPTY), stringUnderTest); + assertSame("empty concatenated with " + classUnderTest + " must give " + classUnderTest, + ByteString.EMPTY.concat(stringUnderTest), stringUnderTest); + } +} diff --git a/java/src/test/java/com/google/protobuf/MessageTest.java b/java/src/test/java/com/google/protobuf/MessageTest.java index c2f47eb2..747fed75 100644 --- a/java/src/test/java/com/google/protobuf/MessageTest.java +++ b/java/src/test/java/com/google/protobuf/MessageTest.java @@ -38,6 +38,8 @@ import protobuf_unittest.UnittestProto.ForeignMessage; import junit.framework.TestCase; +import java.util.List; + /** * Misc. unit tests for message operations that apply to both generated * and dynamic messages. @@ -310,4 +312,42 @@ public class MessageTest extends TestCase { assertEquals("Message missing required fields: a, b, c", e.getMessage()); } } + + /** Test reading unset repeated message from DynamicMessage. */ + public void testDynamicRepeatedMessageNull() throws Exception { + Descriptors.Descriptor descriptor = TestRequired.getDescriptor(); + DynamicMessage result = + DynamicMessage.newBuilder(TestAllTypes.getDescriptor()) + .mergeFrom(DynamicMessage.newBuilder(MERGE_SOURCE).build()) + .build(); + + assertTrue(result.getField(result.getDescriptorForType() + .findFieldByName("repeated_foreign_message")) instanceof List); + assertEquals(result.getRepeatedFieldCount(result.getDescriptorForType() + .findFieldByName("repeated_foreign_message")), 0); + } + + /** Test reading repeated message from DynamicMessage. */ + public void testDynamicRepeatedMessageNotNull() throws Exception { + + TestAllTypes REPEATED_NESTED = + TestAllTypes.newBuilder() + .setOptionalInt32(1) + .setOptionalString("foo") + .setOptionalForeignMessage(ForeignMessage.getDefaultInstance()) + .addRepeatedString("bar") + .addRepeatedForeignMessage(ForeignMessage.getDefaultInstance()) + .addRepeatedForeignMessage(ForeignMessage.getDefaultInstance()) + .build(); + Descriptors.Descriptor descriptor = TestRequired.getDescriptor(); + DynamicMessage result = + DynamicMessage.newBuilder(TestAllTypes.getDescriptor()) + .mergeFrom(DynamicMessage.newBuilder(REPEATED_NESTED).build()) + .build(); + + assertTrue(result.getField(result.getDescriptorForType() + .findFieldByName("repeated_foreign_message")) instanceof List); + assertEquals(result.getRepeatedFieldCount(result.getDescriptorForType() + .findFieldByName("repeated_foreign_message")), 2); + } } diff --git a/java/src/test/java/com/google/protobuf/ParserTest.java b/java/src/test/java/com/google/protobuf/ParserTest.java new file mode 100644 index 00000000..396902cf --- /dev/null +++ b/java/src/test/java/com/google/protobuf/ParserTest.java @@ -0,0 +1,375 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// http://code.google.com/p/protobuf/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import com.google.protobuf.UnittestLite.TestAllTypesLite; +import com.google.protobuf.UnittestLite.TestPackedExtensionsLite; +import com.google.protobuf.UnittestLite.TestParsingMergeLite; +import com.google.protobuf.UnittestLite; +import protobuf_unittest.UnittestOptimizeFor.TestOptimizedForSize; +import protobuf_unittest.UnittestOptimizeFor.TestRequiredOptimizedForSize; +import protobuf_unittest.UnittestOptimizeFor; +import protobuf_unittest.UnittestProto.ForeignMessage; +import protobuf_unittest.UnittestProto.TestAllTypes; +import protobuf_unittest.UnittestProto.TestEmptyMessage; +import protobuf_unittest.UnittestProto.TestRequired; +import protobuf_unittest.UnittestProto.TestParsingMerge; +import protobuf_unittest.UnittestProto; + +import junit.framework.TestCase; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; + +/** + * Unit test for {@link Parser}. + * + * @author liujisi@google.com (Pherl Liu) + */ +public class ParserTest extends TestCase { + public void testGeneratedMessageParserSingleton() throws Exception { + for (int i = 0; i < 10; i++) { + assertEquals(TestAllTypes.PARSER, + TestUtil.getAllSet().getParserForType()); + } + } + + private void assertRoundTripEquals(MessageLite message, + ExtensionRegistryLite registry) + throws Exception { + final byte[] data = message.toByteArray(); + final int offset = 20; + final int length = data.length; + final int padding = 30; + Parser parser = message.getParserForType(); + assertMessageEquals(message, parser.parseFrom(data, registry)); + assertMessageEquals(message, parser.parseFrom( + generatePaddingArray(data, offset, padding), + offset, length, registry)); + assertMessageEquals(message, parser.parseFrom( + message.toByteString(), registry)); + assertMessageEquals(message, parser.parseFrom( + new ByteArrayInputStream(data), registry)); + assertMessageEquals(message, parser.parseFrom( + CodedInputStream.newInstance(data), registry)); + } + + private void assertRoundTripEquals(MessageLite message) throws Exception { + final byte[] data = message.toByteArray(); + final int offset = 20; + final int length = data.length; + final int padding = 30; + Parser parser = message.getParserForType(); + assertMessageEquals(message, parser.parseFrom(data)); + assertMessageEquals(message, parser.parseFrom( + generatePaddingArray(data, offset, padding), + offset, length)); + assertMessageEquals(message, parser.parseFrom(message.toByteString())); + assertMessageEquals(message, parser.parseFrom( + new ByteArrayInputStream(data))); + assertMessageEquals(message, parser.parseFrom( + CodedInputStream.newInstance(data))); + } + + private void assertMessageEquals(MessageLite expected, MessageLite actual) + throws Exception { + if (expected instanceof Message) { + assertEquals(expected, actual); + } else { + assertEquals(expected.toByteString(), actual.toByteString()); + } + } + + private byte[] generatePaddingArray(byte[] data, int offset, int padding) { + byte[] result = new byte[offset + data.length + padding]; + System.arraycopy(data, 0, result, offset, data.length); + return result; + } + + public void testNormalMessage() throws Exception { + assertRoundTripEquals(TestUtil.getAllSet()); + } + + public void testParsePartial() throws Exception { + Parser parser = TestRequired.PARSER; + final String errorString = + "Should throw exceptions when the parsed message isn't initialized."; + + // TestRequired.b and TestRequired.c are not set. + TestRequired partialMessage = TestRequired.newBuilder() + .setA(1).buildPartial(); + + // parsePartialFrom should pass. + byte[] data = partialMessage.toByteArray(); + assertEquals(partialMessage, parser.parsePartialFrom(data)); + assertEquals(partialMessage, parser.parsePartialFrom( + partialMessage.toByteString())); + assertEquals(partialMessage, parser.parsePartialFrom( + new ByteArrayInputStream(data))); + assertEquals(partialMessage, parser.parsePartialFrom( + CodedInputStream.newInstance(data))); + + // parseFrom(ByteArray) + try { + parser.parseFrom(partialMessage.toByteArray()); + fail(errorString); + } catch (InvalidProtocolBufferException e) { + // pass. + } + + // parseFrom(ByteString) + try { + parser.parseFrom(partialMessage.toByteString()); + fail(errorString); + } catch (InvalidProtocolBufferException e) { + // pass. + } + + // parseFrom(InputStream) + try { + parser.parseFrom(new ByteArrayInputStream(partialMessage.toByteArray())); + fail(errorString); + } catch (IOException e) { + // pass. + } + + // parseFrom(CodedInputStream) + try { + parser.parseFrom(CodedInputStream.newInstance( + partialMessage.toByteArray())); + fail(errorString); + } catch (IOException e) { + // pass. + } + } + + public void testParseExtensions() throws Exception { + assertRoundTripEquals(TestUtil.getAllExtensionsSet(), + TestUtil.getExtensionRegistry()); + assertRoundTripEquals(TestUtil.getAllLiteExtensionsSet(), + TestUtil.getExtensionRegistryLite()); + } + + public void testParsePacked() throws Exception { + assertRoundTripEquals(TestUtil.getPackedSet()); + assertRoundTripEquals(TestUtil.getPackedExtensionsSet(), + TestUtil.getExtensionRegistry()); + assertRoundTripEquals(TestUtil.getLitePackedExtensionsSet(), + TestUtil.getExtensionRegistryLite()); + } + + public void testParseDelimitedTo() throws Exception { + // Write normal Message. + TestAllTypes normalMessage = TestUtil.getAllSet(); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + normalMessage.writeDelimitedTo(output); + + // Write MessageLite with packed extension fields. + TestPackedExtensionsLite packedMessage = + TestUtil.getLitePackedExtensionsSet(); + packedMessage.writeDelimitedTo(output); + + InputStream input = new ByteArrayInputStream(output.toByteArray()); + assertMessageEquals( + normalMessage, + normalMessage.getParserForType().parseDelimitedFrom(input)); + assertMessageEquals( + packedMessage, + packedMessage.getParserForType().parseDelimitedFrom( + input, TestUtil.getExtensionRegistryLite())); + } + + public void testParseUnknownFields() throws Exception { + // All fields will be treated as unknown fields in emptyMessage. + TestEmptyMessage emptyMessage = TestEmptyMessage.PARSER.parseFrom( + TestUtil.getAllSet().toByteString()); + assertEquals( + TestUtil.getAllSet().toByteString(), + emptyMessage.toByteString()); + } + + public void testOptimizeForSize() throws Exception { + TestOptimizedForSize.Builder builder = TestOptimizedForSize.newBuilder(); + builder.setI(12).setMsg(ForeignMessage.newBuilder().setC(34).build()); + builder.setExtension(TestOptimizedForSize.testExtension, 56); + builder.setExtension(TestOptimizedForSize.testExtension2, + TestRequiredOptimizedForSize.newBuilder().setX(78).build()); + + TestOptimizedForSize message = builder.build(); + ExtensionRegistry registry = ExtensionRegistry.newInstance(); + UnittestOptimizeFor.registerAllExtensions(registry); + + assertRoundTripEquals(message, registry); + } + + /** Helper method for {@link #testParsingMerge()}.*/ + private void assertMessageMerged(TestAllTypes allTypes) + throws Exception { + assertEquals(3, allTypes.getOptionalInt32()); + assertEquals(2, allTypes.getOptionalInt64()); + assertEquals("hello", allTypes.getOptionalString()); + } + + /** Helper method for {@link #testParsingMergeLite()}.*/ + private void assertMessageMerged(TestAllTypesLite allTypes) + throws Exception { + assertEquals(3, allTypes.getOptionalInt32()); + assertEquals(2, allTypes.getOptionalInt64()); + assertEquals("hello", allTypes.getOptionalString()); + } + + public void testParsingMerge() throws Exception { + // Build messages. + TestAllTypes.Builder builder = TestAllTypes.newBuilder(); + TestAllTypes msg1 = builder.setOptionalInt32(1).build(); + builder.clear(); + TestAllTypes msg2 = builder.setOptionalInt64(2).build(); + builder.clear(); + TestAllTypes msg3 = builder.setOptionalInt32(3) + .setOptionalString("hello").build(); + + // Build groups. + TestParsingMerge.RepeatedFieldsGenerator.Group1 optionalG1 = + TestParsingMerge.RepeatedFieldsGenerator.Group1.newBuilder() + .setField1(msg1).build(); + TestParsingMerge.RepeatedFieldsGenerator.Group1 optionalG2 = + TestParsingMerge.RepeatedFieldsGenerator.Group1.newBuilder() + .setField1(msg2).build(); + TestParsingMerge.RepeatedFieldsGenerator.Group1 optionalG3 = + TestParsingMerge.RepeatedFieldsGenerator.Group1.newBuilder() + .setField1(msg3).build(); + TestParsingMerge.RepeatedFieldsGenerator.Group2 repeatedG1 = + TestParsingMerge.RepeatedFieldsGenerator.Group2.newBuilder() + .setField1(msg1).build(); + TestParsingMerge.RepeatedFieldsGenerator.Group2 repeatedG2 = + TestParsingMerge.RepeatedFieldsGenerator.Group2.newBuilder() + .setField1(msg2).build(); + TestParsingMerge.RepeatedFieldsGenerator.Group2 repeatedG3 = + TestParsingMerge.RepeatedFieldsGenerator.Group2.newBuilder() + .setField1(msg3).build(); + + // Assign and serialize RepeatedFieldsGenerator. + ByteString data = TestParsingMerge.RepeatedFieldsGenerator.newBuilder() + .addField1(msg1).addField1(msg2).addField1(msg3) + .addField2(msg1).addField2(msg2).addField2(msg3) + .addField3(msg1).addField3(msg2).addField3(msg3) + .addGroup1(optionalG1).addGroup1(optionalG2).addGroup1(optionalG3) + .addGroup2(repeatedG1).addGroup2(repeatedG2).addGroup2(repeatedG3) + .addExt1(msg1).addExt1(msg2).addExt1(msg3) + .addExt2(msg1).addExt2(msg2).addExt2(msg3) + .build().toByteString(); + + // Parse TestParsingMerge. + ExtensionRegistry registry = ExtensionRegistry.newInstance(); + UnittestProto.registerAllExtensions(registry); + TestParsingMerge parsingMerge = + TestParsingMerge.PARSER.parseFrom(data, registry); + + // Required and optional fields should be merged. + assertMessageMerged(parsingMerge.getRequiredAllTypes()); + assertMessageMerged(parsingMerge.getOptionalAllTypes()); + assertMessageMerged( + parsingMerge.getOptionalGroup().getOptionalGroupAllTypes()); + assertMessageMerged(parsingMerge.getExtension( + TestParsingMerge.optionalExt)); + + // Repeated fields should not be merged. + assertEquals(3, parsingMerge.getRepeatedAllTypesCount()); + assertEquals(3, parsingMerge.getRepeatedGroupCount()); + assertEquals(3, parsingMerge.getExtensionCount( + TestParsingMerge.repeatedExt)); + } + + public void testParsingMergeLite() throws Exception { + // Build messages. + TestAllTypesLite.Builder builder = + TestAllTypesLite.newBuilder(); + TestAllTypesLite msg1 = builder.setOptionalInt32(1).build(); + builder.clear(); + TestAllTypesLite msg2 = builder.setOptionalInt64(2).build(); + builder.clear(); + TestAllTypesLite msg3 = builder.setOptionalInt32(3) + .setOptionalString("hello").build(); + + // Build groups. + TestParsingMergeLite.RepeatedFieldsGenerator.Group1 optionalG1 = + TestParsingMergeLite.RepeatedFieldsGenerator.Group1.newBuilder() + .setField1(msg1).build(); + TestParsingMergeLite.RepeatedFieldsGenerator.Group1 optionalG2 = + TestParsingMergeLite.RepeatedFieldsGenerator.Group1.newBuilder() + .setField1(msg2).build(); + TestParsingMergeLite.RepeatedFieldsGenerator.Group1 optionalG3 = + TestParsingMergeLite.RepeatedFieldsGenerator.Group1.newBuilder() + .setField1(msg3).build(); + TestParsingMergeLite.RepeatedFieldsGenerator.Group2 repeatedG1 = + TestParsingMergeLite.RepeatedFieldsGenerator.Group2.newBuilder() + .setField1(msg1).build(); + TestParsingMergeLite.RepeatedFieldsGenerator.Group2 repeatedG2 = + TestParsingMergeLite.RepeatedFieldsGenerator.Group2.newBuilder() + .setField1(msg2).build(); + TestParsingMergeLite.RepeatedFieldsGenerator.Group2 repeatedG3 = + TestParsingMergeLite.RepeatedFieldsGenerator.Group2.newBuilder() + .setField1(msg3).build(); + + // Assign and serialize RepeatedFieldsGenerator. + ByteString data = TestParsingMergeLite.RepeatedFieldsGenerator.newBuilder() + .addField1(msg1).addField1(msg2).addField1(msg3) + .addField2(msg1).addField2(msg2).addField2(msg3) + .addField3(msg1).addField3(msg2).addField3(msg3) + .addGroup1(optionalG1).addGroup1(optionalG2).addGroup1(optionalG3) + .addGroup2(repeatedG1).addGroup2(repeatedG2).addGroup2(repeatedG3) + .addExt1(msg1).addExt1(msg2).addExt1(msg3) + .addExt2(msg1).addExt2(msg2).addExt2(msg3) + .build().toByteString(); + + // Parse TestParsingMergeLite. + ExtensionRegistry registry = ExtensionRegistry.newInstance(); + UnittestLite.registerAllExtensions(registry); + TestParsingMergeLite parsingMerge = + TestParsingMergeLite.PARSER.parseFrom(data, registry); + + // Required and optional fields should be merged. + assertMessageMerged(parsingMerge.getRequiredAllTypes()); + assertMessageMerged(parsingMerge.getOptionalAllTypes()); + assertMessageMerged( + parsingMerge.getOptionalGroup().getOptionalGroupAllTypes()); + assertMessageMerged(parsingMerge.getExtension( + TestParsingMergeLite.optionalExt)); + + // Repeated fields should not be merged. + assertEquals(3, parsingMerge.getRepeatedAllTypesCount()); + assertEquals(3, parsingMerge.getRepeatedGroupCount()); + assertEquals(3, parsingMerge.getExtensionCount( + TestParsingMergeLite.repeatedExt)); + } +} diff --git a/java/src/test/java/com/google/protobuf/RopeByteStringSubstringTest.java b/java/src/test/java/com/google/protobuf/RopeByteStringSubstringTest.java new file mode 100644 index 00000000..06707a2f --- /dev/null +++ b/java/src/test/java/com/google/protobuf/RopeByteStringSubstringTest.java @@ -0,0 +1,97 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// http://code.google.com/p/protobuf/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import java.io.UnsupportedEncodingException; +import java.util.Iterator; + +/** + * This class tests {@link RopeByteString#substring(int, int)} by inheriting the tests from + * {@link LiteralByteStringTest}. Only a couple of methods are overridden. + * + * @author carlanton@google.com (Carl Haverl) + */ +public class RopeByteStringSubstringTest extends LiteralByteStringTest { + + @Override + protected void setUp() throws Exception { + classUnderTest = "RopeByteString"; + byte[] sourceBytes = ByteStringTest.getTestBytes(22341, 22337766L); + Iterator iter = ByteStringTest.makeConcretePieces(sourceBytes).iterator(); + ByteString sourceString = iter.next(); + while (iter.hasNext()) { + sourceString = sourceString.concat(iter.next()); + } + + int from = 1130; + int to = sourceBytes.length - 5555; + stringUnderTest = sourceString.substring(from, to); + referenceBytes = new byte[to - from]; + System.arraycopy(sourceBytes, from, referenceBytes, 0, to - from); + expectedHashCode = -1259260680; + } + + @Override + public void testGetTreeDepth() { + assertEquals(classUnderTest + " must have the expected tree depth", + 3, stringUnderTest.getTreeDepth()); + } + + @Override + public void testToString() throws UnsupportedEncodingException { + String sourceString = "I love unicode \u1234\u5678 characters"; + ByteString sourceByteString = ByteString.copyFromUtf8(sourceString); + int copies = 250; + + // By building the RopeByteString by concatenating, this is actually a fairly strenuous test. + StringBuilder builder = new StringBuilder(copies * sourceString.length()); + ByteString unicode = ByteString.EMPTY; + for (int i = 0; i < copies; ++i) { + builder.append(sourceString); + unicode = RopeByteString.concatenate(unicode, sourceByteString); + } + String testString = builder.toString(); + + // Do the substring part + testString = testString.substring(2, testString.length() - 6); + unicode = unicode.substring(2, unicode.size() - 6); + + assertEquals(classUnderTest + " from string must have the expected type", + classUnderTest, getActualClassName(unicode)); + String roundTripString = unicode.toString(UTF_8); + assertEquals(classUnderTest + " unicode bytes must match", + testString, roundTripString); + ByteString flatString = ByteString.copyFromUtf8(testString); + assertEquals(classUnderTest + " string must equal the flat string", flatString, unicode); + assertEquals(classUnderTest + " string must must have same hashCode as the flat string", + flatString.hashCode(), unicode.hashCode()); + } +} diff --git a/java/src/test/java/com/google/protobuf/RopeByteStringTest.java b/java/src/test/java/com/google/protobuf/RopeByteStringTest.java new file mode 100644 index 00000000..15f660d9 --- /dev/null +++ b/java/src/test/java/com/google/protobuf/RopeByteStringTest.java @@ -0,0 +1,115 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// http://code.google.com/p/protobuf/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import java.io.UnsupportedEncodingException; +import java.util.Arrays; +import java.util.Iterator; + +/** + * This class tests {@link RopeByteString} by inheriting the tests from + * {@link LiteralByteStringTest}. Only a couple of methods are overridden. + * + *

A full test of the result of {@link RopeByteString#substring(int, int)} is found in the + * separate class {@link RopeByteStringSubstringTest}. + * + * @author carlanton@google.com (Carl Haverl) + */ +public class RopeByteStringTest extends LiteralByteStringTest { + + @Override + protected void setUp() throws Exception { + classUnderTest = "RopeByteString"; + referenceBytes = ByteStringTest.getTestBytes(22341, 22337766L); + Iterator iter = ByteStringTest.makeConcretePieces(referenceBytes).iterator(); + stringUnderTest = iter.next(); + while (iter.hasNext()) { + stringUnderTest = stringUnderTest.concat(iter.next()); + } + expectedHashCode = -1214197238; + } + + @Override + public void testGetTreeDepth() { + assertEquals(classUnderTest + " must have the expected tree depth", + 4, stringUnderTest.getTreeDepth()); + } + + public void testBalance() { + int numberOfPieces = 10000; + int pieceSize = 64; + byte[] testBytes = ByteStringTest.getTestBytes(numberOfPieces * pieceSize, 113377L); + + // Build up a big ByteString from smaller pieces to force a rebalance + ByteString concatenated = ByteString.EMPTY; + for (int i = 0; i < numberOfPieces; ++i) { + concatenated = concatenated.concat(ByteString.copyFrom(testBytes, i * pieceSize, pieceSize)); + } + + assertEquals(classUnderTest + " from string must have the expected type", + classUnderTest, getActualClassName(concatenated)); + assertTrue(classUnderTest + " underlying bytes must match after balancing", + Arrays.equals(testBytes, concatenated.toByteArray())); + ByteString testString = ByteString.copyFrom(testBytes); + assertTrue(classUnderTest + " balanced string must equal flat string", + concatenated.equals(testString)); + assertTrue(classUnderTest + " flat string must equal balanced string", + testString.equals(concatenated)); + assertEquals(classUnderTest + " balanced string must have same hash code as flat string", + testString.hashCode(), concatenated.hashCode()); + } + + @Override + public void testToString() throws UnsupportedEncodingException { + String sourceString = "I love unicode \u1234\u5678 characters"; + ByteString sourceByteString = ByteString.copyFromUtf8(sourceString); + int copies = 250; + + // By building the RopeByteString by concatenating, this is actually a fairly strenuous test. + StringBuilder builder = new StringBuilder(copies * sourceString.length()); + ByteString unicode = ByteString.EMPTY; + for (int i = 0; i < copies; ++i) { + builder.append(sourceString); + unicode = RopeByteString.concatenate(unicode, sourceByteString); + } + String testString = builder.toString(); + + assertEquals(classUnderTest + " from string must have the expected type", + classUnderTest, getActualClassName(unicode)); + String roundTripString = unicode.toString(UTF_8); + assertEquals(classUnderTest + " unicode bytes must match", + testString, roundTripString); + ByteString flatString = ByteString.copyFromUtf8(testString); + assertEquals(classUnderTest + " string must equal the flat string", flatString, unicode); + assertEquals(classUnderTest + " string must must have same hashCode as the flat string", + flatString.hashCode(), unicode.hashCode()); + } +} diff --git a/java/src/test/java/com/google/protobuf/TestBadIdentifiers.java b/java/src/test/java/com/google/protobuf/TestBadIdentifiers.java index 6feec4e6..382acf0f 100644 --- a/java/src/test/java/com/google/protobuf/TestBadIdentifiers.java +++ b/java/src/test/java/com/google/protobuf/TestBadIdentifiers.java @@ -35,7 +35,7 @@ import junit.framework.TestCase; /** * Tests that proto2 api generation doesn't cause compile errors when * compiling protocol buffers that have names that would otherwise conflict - * if not fully qualified (like @Deprecated and @Override). + * if not fully qualified (like @Deprecated and @Override). * * @author jonp@google.com (Jon Perlow) */ @@ -45,5 +45,19 @@ public class TestBadIdentifiers extends TestCase { // If this compiles, it means the generation was correct. TestBadIdentifiersProto.Deprecated.newBuilder(); TestBadIdentifiersProto.Override.newBuilder(); - } + } + + public void testGetDescriptor() { + Descriptors.FileDescriptor fileDescriptor = + TestBadIdentifiersProto.getDescriptor(); + String descriptorField = TestBadIdentifiersProto.Descriptor + .getDefaultInstance().getDescriptor(); + Descriptors.Descriptor protoDescriptor = TestBadIdentifiersProto.Descriptor + .getDefaultInstance().getDescriptorForType(); + String nestedDescriptorField = TestBadIdentifiersProto.Descriptor + .NestedDescriptor.getDefaultInstance().getDescriptor(); + Descriptors.Descriptor nestedProtoDescriptor = TestBadIdentifiersProto + .Descriptor.NestedDescriptor.getDefaultInstance() + .getDescriptorForType(); + } } diff --git a/java/src/test/java/com/google/protobuf/TestUtil.java b/java/src/test/java/com/google/protobuf/TestUtil.java index 9109f419..76f5c602 100644 --- a/java/src/test/java/com/google/protobuf/TestUtil.java +++ b/java/src/test/java/com/google/protobuf/TestUtil.java @@ -72,14 +72,16 @@ import static protobuf_unittest.UnittestProto.optionalBoolExtension; import static protobuf_unittest.UnittestProto.optionalStringExtension; import static protobuf_unittest.UnittestProto.optionalBytesExtension; import static protobuf_unittest.UnittestProto.optionalGroupExtension; -import static protobuf_unittest.UnittestProto.optionalNestedMessageExtension; +import static protobuf_unittest.UnittestProto.optionalCordExtension; +import static protobuf_unittest.UnittestProto.optionalForeignEnumExtension; import static protobuf_unittest.UnittestProto.optionalForeignMessageExtension; +import static protobuf_unittest.UnittestProto.optionalImportEnumExtension; import static protobuf_unittest.UnittestProto.optionalImportMessageExtension; import static protobuf_unittest.UnittestProto.optionalNestedEnumExtension; -import static protobuf_unittest.UnittestProto.optionalForeignEnumExtension; -import static protobuf_unittest.UnittestProto.optionalImportEnumExtension; +import static protobuf_unittest.UnittestProto.optionalNestedMessageExtension; +import static protobuf_unittest.UnittestProto.optionalPublicImportMessageExtension; +import static protobuf_unittest.UnittestProto.optionalLazyMessageExtension; import static protobuf_unittest.UnittestProto.optionalStringPieceExtension; -import static protobuf_unittest.UnittestProto.optionalCordExtension; import static protobuf_unittest.UnittestProto.repeatedInt32Extension; import static protobuf_unittest.UnittestProto.repeatedInt64Extension; @@ -100,6 +102,7 @@ import static protobuf_unittest.UnittestProto.repeatedGroupExtension; import static protobuf_unittest.UnittestProto.repeatedNestedMessageExtension; import static protobuf_unittest.UnittestProto.repeatedForeignMessageExtension; import static protobuf_unittest.UnittestProto.repeatedImportMessageExtension; +import static protobuf_unittest.UnittestProto.repeatedLazyMessageExtension; import static protobuf_unittest.UnittestProto.repeatedNestedEnumExtension; import static protobuf_unittest.UnittestProto.repeatedForeignEnumExtension; import static protobuf_unittest.UnittestProto.repeatedImportEnumExtension; @@ -162,11 +165,13 @@ import static com.google.protobuf.UnittestLite.optionalStringExtensionLite; import static com.google.protobuf.UnittestLite.optionalBytesExtensionLite; import static com.google.protobuf.UnittestLite.optionalGroupExtensionLite; import static com.google.protobuf.UnittestLite.optionalNestedMessageExtensionLite; +import static com.google.protobuf.UnittestLite.optionalForeignEnumExtensionLite; import static com.google.protobuf.UnittestLite.optionalForeignMessageExtensionLite; +import static com.google.protobuf.UnittestLite.optionalImportEnumExtensionLite; import static com.google.protobuf.UnittestLite.optionalImportMessageExtensionLite; import static com.google.protobuf.UnittestLite.optionalNestedEnumExtensionLite; -import static com.google.protobuf.UnittestLite.optionalForeignEnumExtensionLite; -import static com.google.protobuf.UnittestLite.optionalImportEnumExtensionLite; +import static com.google.protobuf.UnittestLite.optionalPublicImportMessageExtensionLite; +import static com.google.protobuf.UnittestLite.optionalLazyMessageExtensionLite; import static com.google.protobuf.UnittestLite.optionalStringPieceExtensionLite; import static com.google.protobuf.UnittestLite.optionalCordExtensionLite; @@ -189,6 +194,7 @@ import static com.google.protobuf.UnittestLite.repeatedGroupExtensionLite; import static com.google.protobuf.UnittestLite.repeatedNestedMessageExtensionLite; import static com.google.protobuf.UnittestLite.repeatedForeignMessageExtensionLite; import static com.google.protobuf.UnittestLite.repeatedImportMessageExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedLazyMessageExtensionLite; import static com.google.protobuf.UnittestLite.repeatedNestedEnumExtensionLite; import static com.google.protobuf.UnittestLite.repeatedForeignEnumExtensionLite; import static com.google.protobuf.UnittestLite.repeatedImportEnumExtensionLite; @@ -222,8 +228,9 @@ import protobuf_unittest.UnittestProto.TestPackedTypes; import protobuf_unittest.UnittestProto.TestUnpackedTypes; import protobuf_unittest.UnittestProto.ForeignMessage; import protobuf_unittest.UnittestProto.ForeignEnum; -import com.google.protobuf.test.UnittestImport.ImportMessage; import com.google.protobuf.test.UnittestImport.ImportEnum; +import com.google.protobuf.test.UnittestImport.ImportMessage; +import com.google.protobuf.test.UnittestImportPublic.PublicImportMessage; import com.google.protobuf.UnittestLite.TestAllTypesLite; import com.google.protobuf.UnittestLite.TestAllExtensionsLite; @@ -231,8 +238,9 @@ import com.google.protobuf.UnittestLite.TestAllExtensionsLiteOrBuilder; import com.google.protobuf.UnittestLite.TestPackedExtensionsLite; import com.google.protobuf.UnittestLite.ForeignMessageLite; import com.google.protobuf.UnittestLite.ForeignEnumLite; -import com.google.protobuf.UnittestImportLite.ImportMessageLite; import com.google.protobuf.UnittestImportLite.ImportEnumLite; +import com.google.protobuf.UnittestImportLite.ImportMessageLite; +import com.google.protobuf.UnittestImportPublicLite.PublicImportMessageLite; import junit.framework.Assert; @@ -242,7 +250,7 @@ import java.io.RandomAccessFile; /** * Contains methods for setting all fields of {@code TestAllTypes} to - * some vaules as well as checking that all the fields are set to those values. + * some values as well as checking that all the fields are set to those values. * These are useful for testing various protocol message features, e.g. * set all fields of a message, serialize it, parse it, and check that all * fields are set. @@ -274,6 +282,16 @@ public final class TestUtil { return builder.build(); } + /** + * Get a {@code TestAllTypes.Builder} with all fields set as they would be by + * {@link #setAllFields(TestAllTypes.Builder)}. + */ + public static TestAllTypes.Builder getAllSetBuilder() { + TestAllTypes.Builder builder = TestAllTypes.newBuilder(); + setAllFields(builder); + return builder; + } + /** * Get a {@code TestAllExtensions} with all fields set as they would be by * {@link #setAllExtensions(TestAllExtensions.Builder)}. @@ -344,6 +362,10 @@ public final class TestUtil { ForeignMessage.newBuilder().setC(119).build()); message.setOptionalImportMessage( ImportMessage.newBuilder().setD(120).build()); + message.setOptionalPublicImportMessage( + PublicImportMessage.newBuilder().setE(126).build()); + message.setOptionalLazyMessage( + TestAllTypes.NestedMessage.newBuilder().setBb(127).build()); message.setOptionalNestedEnum (TestAllTypes.NestedEnum.BAZ); message.setOptionalForeignEnum(ForeignEnum.FOREIGN_BAZ); @@ -378,6 +400,8 @@ public final class TestUtil { ForeignMessage.newBuilder().setC(219).build()); message.addRepeatedImportMessage( ImportMessage.newBuilder().setD(220).build()); + message.addRepeatedLazyMessage( + TestAllTypes.NestedMessage.newBuilder().setBb(227).build()); message.addRepeatedNestedEnum (TestAllTypes.NestedEnum.BAR); message.addRepeatedForeignEnum(ForeignEnum.FOREIGN_BAR); @@ -411,6 +435,8 @@ public final class TestUtil { ForeignMessage.newBuilder().setC(319).build()); message.addRepeatedImportMessage( ImportMessage.newBuilder().setD(320).build()); + message.addRepeatedLazyMessage( + TestAllTypes.NestedMessage.newBuilder().setBb(327).build()); message.addRepeatedNestedEnum (TestAllTypes.NestedEnum.BAZ); message.addRepeatedForeignEnum(ForeignEnum.FOREIGN_BAZ); @@ -476,6 +502,8 @@ public final class TestUtil { ForeignMessage.newBuilder().setC(519).build()); message.setRepeatedImportMessage(1, ImportMessage.newBuilder().setD(520).build()); + message.setRepeatedLazyMessage(1, + TestAllTypes.NestedMessage.newBuilder().setBb(527).build()); message.setRepeatedNestedEnum (1, TestAllTypes.NestedEnum.FOO); message.setRepeatedForeignEnum(1, ForeignEnum.FOREIGN_FOO); @@ -541,10 +569,12 @@ public final class TestUtil { Assert.assertEquals("115", message.getOptionalString ()); Assert.assertEquals(toBytes("116"), message.getOptionalBytes()); - Assert.assertEquals(117, message.getOptionalGroup ().getA()); - Assert.assertEquals(118, message.getOptionalNestedMessage ().getBb()); - Assert.assertEquals(119, message.getOptionalForeignMessage().getC()); - Assert.assertEquals(120, message.getOptionalImportMessage ().getD()); + Assert.assertEquals(117, message.getOptionalGroup ().getA()); + Assert.assertEquals(118, message.getOptionalNestedMessage ().getBb()); + Assert.assertEquals(119, message.getOptionalForeignMessage ().getC()); + Assert.assertEquals(120, message.getOptionalImportMessage ().getD()); + Assert.assertEquals(126, message.getOptionalPublicImportMessage().getE()); + Assert.assertEquals(127, message.getOptionalLazyMessage ().getBb()); Assert.assertEquals(TestAllTypes.NestedEnum.BAZ, message.getOptionalNestedEnum()); Assert.assertEquals(ForeignEnum.FOREIGN_BAZ, message.getOptionalForeignEnum()); @@ -575,6 +605,7 @@ public final class TestUtil { Assert.assertEquals(2, message.getRepeatedNestedMessageCount ()); Assert.assertEquals(2, message.getRepeatedForeignMessageCount()); Assert.assertEquals(2, message.getRepeatedImportMessageCount ()); + Assert.assertEquals(2, message.getRepeatedLazyMessageCount ()); Assert.assertEquals(2, message.getRepeatedNestedEnumCount ()); Assert.assertEquals(2, message.getRepeatedForeignEnumCount ()); Assert.assertEquals(2, message.getRepeatedImportEnumCount ()); @@ -602,6 +633,7 @@ public final class TestUtil { Assert.assertEquals(218, message.getRepeatedNestedMessage (0).getBb()); Assert.assertEquals(219, message.getRepeatedForeignMessage(0).getC()); Assert.assertEquals(220, message.getRepeatedImportMessage (0).getD()); + Assert.assertEquals(227, message.getRepeatedLazyMessage (0).getBb()); Assert.assertEquals(TestAllTypes.NestedEnum.BAR, message.getRepeatedNestedEnum (0)); Assert.assertEquals(ForeignEnum.FOREIGN_BAR, message.getRepeatedForeignEnum(0)); @@ -630,6 +662,7 @@ public final class TestUtil { Assert.assertEquals(318, message.getRepeatedNestedMessage (1).getBb()); Assert.assertEquals(319, message.getRepeatedForeignMessage(1).getC()); Assert.assertEquals(320, message.getRepeatedImportMessage (1).getD()); + Assert.assertEquals(327, message.getRepeatedLazyMessage (1).getBb()); Assert.assertEquals(TestAllTypes.NestedEnum.BAZ, message.getRepeatedNestedEnum (1)); Assert.assertEquals(ForeignEnum.FOREIGN_BAZ, message.getRepeatedForeignEnum(1)); @@ -741,15 +774,19 @@ public final class TestUtil { Assert.assertEquals(ByteString.EMPTY, message.getOptionalBytes()); // Embedded messages should also be clear. - Assert.assertFalse(message.getOptionalGroup ().hasA()); - Assert.assertFalse(message.getOptionalNestedMessage ().hasBb()); - Assert.assertFalse(message.getOptionalForeignMessage().hasC()); - Assert.assertFalse(message.getOptionalImportMessage ().hasD()); - - Assert.assertEquals(0, message.getOptionalGroup ().getA()); - Assert.assertEquals(0, message.getOptionalNestedMessage ().getBb()); - Assert.assertEquals(0, message.getOptionalForeignMessage().getC()); - Assert.assertEquals(0, message.getOptionalImportMessage ().getD()); + Assert.assertFalse(message.getOptionalGroup ().hasA()); + Assert.assertFalse(message.getOptionalNestedMessage ().hasBb()); + Assert.assertFalse(message.getOptionalForeignMessage ().hasC()); + Assert.assertFalse(message.getOptionalImportMessage ().hasD()); + Assert.assertFalse(message.getOptionalPublicImportMessage().hasE()); + Assert.assertFalse(message.getOptionalLazyMessage ().hasBb()); + + Assert.assertEquals(0, message.getOptionalGroup ().getA()); + Assert.assertEquals(0, message.getOptionalNestedMessage ().getBb()); + Assert.assertEquals(0, message.getOptionalForeignMessage ().getC()); + Assert.assertEquals(0, message.getOptionalImportMessage ().getD()); + Assert.assertEquals(0, message.getOptionalPublicImportMessage().getE()); + Assert.assertEquals(0, message.getOptionalLazyMessage ().getBb()); // Enums without defaults are set to the first value in the enum. Assert.assertEquals(TestAllTypes.NestedEnum.FOO, message.getOptionalNestedEnum ()); @@ -780,6 +817,7 @@ public final class TestUtil { Assert.assertEquals(0, message.getRepeatedNestedMessageCount ()); Assert.assertEquals(0, message.getRepeatedForeignMessageCount()); Assert.assertEquals(0, message.getRepeatedImportMessageCount ()); + Assert.assertEquals(0, message.getRepeatedLazyMessageCount ()); Assert.assertEquals(0, message.getRepeatedNestedEnumCount ()); Assert.assertEquals(0, message.getRepeatedForeignEnumCount ()); Assert.assertEquals(0, message.getRepeatedImportEnumCount ()); @@ -868,6 +906,7 @@ public final class TestUtil { Assert.assertEquals(2, message.getRepeatedNestedMessageCount ()); Assert.assertEquals(2, message.getRepeatedForeignMessageCount()); Assert.assertEquals(2, message.getRepeatedImportMessageCount ()); + Assert.assertEquals(2, message.getRepeatedLazyMessageCount ()); Assert.assertEquals(2, message.getRepeatedNestedEnumCount ()); Assert.assertEquals(2, message.getRepeatedForeignEnumCount ()); Assert.assertEquals(2, message.getRepeatedImportEnumCount ()); @@ -895,6 +934,7 @@ public final class TestUtil { Assert.assertEquals(218, message.getRepeatedNestedMessage (0).getBb()); Assert.assertEquals(219, message.getRepeatedForeignMessage(0).getC()); Assert.assertEquals(220, message.getRepeatedImportMessage (0).getD()); + Assert.assertEquals(227, message.getRepeatedLazyMessage (0).getBb()); Assert.assertEquals(TestAllTypes.NestedEnum.BAR, message.getRepeatedNestedEnum (0)); Assert.assertEquals(ForeignEnum.FOREIGN_BAR, message.getRepeatedForeignEnum(0)); @@ -924,6 +964,7 @@ public final class TestUtil { Assert.assertEquals(518, message.getRepeatedNestedMessage (1).getBb()); Assert.assertEquals(519, message.getRepeatedForeignMessage(1).getC()); Assert.assertEquals(520, message.getRepeatedImportMessage (1).getD()); + Assert.assertEquals(527, message.getRepeatedLazyMessage (1).getBb()); Assert.assertEquals(TestAllTypes.NestedEnum.FOO, message.getRepeatedNestedEnum (1)); Assert.assertEquals(ForeignEnum.FOREIGN_FOO, message.getRepeatedForeignEnum(1)); @@ -1210,6 +1251,10 @@ public final class TestUtil { ForeignMessage.newBuilder().setC(119).build()); message.setExtension(optionalImportMessageExtension, ImportMessage.newBuilder().setD(120).build()); + message.setExtension(optionalPublicImportMessageExtension, + PublicImportMessage.newBuilder().setE(126).build()); + message.setExtension(optionalLazyMessageExtension, + TestAllTypes.NestedMessage.newBuilder().setBb(127).build()); message.setExtension(optionalNestedEnumExtension, TestAllTypes.NestedEnum.BAZ); message.setExtension(optionalForeignEnumExtension, ForeignEnum.FOREIGN_BAZ); @@ -1244,6 +1289,8 @@ public final class TestUtil { ForeignMessage.newBuilder().setC(219).build()); message.addExtension(repeatedImportMessageExtension, ImportMessage.newBuilder().setD(220).build()); + message.addExtension(repeatedLazyMessageExtension, + TestAllTypes.NestedMessage.newBuilder().setBb(227).build()); message.addExtension(repeatedNestedEnumExtension, TestAllTypes.NestedEnum.BAR); message.addExtension(repeatedForeignEnumExtension, ForeignEnum.FOREIGN_BAR); @@ -1277,6 +1324,8 @@ public final class TestUtil { ForeignMessage.newBuilder().setC(319).build()); message.addExtension(repeatedImportMessageExtension, ImportMessage.newBuilder().setD(320).build()); + message.addExtension(repeatedLazyMessageExtension, + TestAllTypes.NestedMessage.newBuilder().setBb(327).build()); message.addExtension(repeatedNestedEnumExtension, TestAllTypes.NestedEnum.BAZ); message.addExtension(repeatedForeignEnumExtension, ForeignEnum.FOREIGN_BAZ); @@ -1343,6 +1392,8 @@ public final class TestUtil { ForeignMessage.newBuilder().setC(519).build()); message.setExtension(repeatedImportMessageExtension, 1, ImportMessage.newBuilder().setD(520).build()); + message.setExtension(repeatedLazyMessageExtension, 1, + TestAllTypes.NestedMessage.newBuilder().setBb(527).build()); message.setExtension(repeatedNestedEnumExtension , 1, TestAllTypes.NestedEnum.FOO); message.setExtension(repeatedForeignEnumExtension, 1, ForeignEnum.FOREIGN_FOO); @@ -1409,10 +1460,12 @@ public final class TestUtil { assertEqualsExactType("115", message.getExtension(optionalStringExtension )); assertEqualsExactType(toBytes("116"), message.getExtension(optionalBytesExtension)); - assertEqualsExactType(117, message.getExtension(optionalGroupExtension ).getA()); - assertEqualsExactType(118, message.getExtension(optionalNestedMessageExtension ).getBb()); - assertEqualsExactType(119, message.getExtension(optionalForeignMessageExtension).getC()); - assertEqualsExactType(120, message.getExtension(optionalImportMessageExtension ).getD()); + assertEqualsExactType(117, message.getExtension(optionalGroupExtension ).getA()); + assertEqualsExactType(118, message.getExtension(optionalNestedMessageExtension ).getBb()); + assertEqualsExactType(119, message.getExtension(optionalForeignMessageExtension ).getC()); + assertEqualsExactType(120, message.getExtension(optionalImportMessageExtension ).getD()); + assertEqualsExactType(126, message.getExtension(optionalPublicImportMessageExtension).getE()); + assertEqualsExactType(127, message.getExtension(optionalLazyMessageExtension ).getBb()); assertEqualsExactType(TestAllTypes.NestedEnum.BAZ, message.getExtension(optionalNestedEnumExtension)); @@ -1446,6 +1499,7 @@ public final class TestUtil { Assert.assertEquals(2, message.getExtensionCount(repeatedNestedMessageExtension )); Assert.assertEquals(2, message.getExtensionCount(repeatedForeignMessageExtension)); Assert.assertEquals(2, message.getExtensionCount(repeatedImportMessageExtension )); + Assert.assertEquals(2, message.getExtensionCount(repeatedLazyMessageExtension )); Assert.assertEquals(2, message.getExtensionCount(repeatedNestedEnumExtension )); Assert.assertEquals(2, message.getExtensionCount(repeatedForeignEnumExtension )); Assert.assertEquals(2, message.getExtensionCount(repeatedImportEnumExtension )); @@ -1473,6 +1527,7 @@ public final class TestUtil { assertEqualsExactType(218, message.getExtension(repeatedNestedMessageExtension , 0).getBb()); assertEqualsExactType(219, message.getExtension(repeatedForeignMessageExtension, 0).getC()); assertEqualsExactType(220, message.getExtension(repeatedImportMessageExtension , 0).getD()); + assertEqualsExactType(227, message.getExtension(repeatedLazyMessageExtension , 0).getBb()); assertEqualsExactType(TestAllTypes.NestedEnum.BAR, message.getExtension(repeatedNestedEnumExtension, 0)); @@ -1504,6 +1559,7 @@ public final class TestUtil { assertEqualsExactType(318, message.getExtension(repeatedNestedMessageExtension , 1).getBb()); assertEqualsExactType(319, message.getExtension(repeatedForeignMessageExtension, 1).getC()); assertEqualsExactType(320, message.getExtension(repeatedImportMessageExtension , 1).getD()); + assertEqualsExactType(327, message.getExtension(repeatedLazyMessageExtension , 1).getBb()); assertEqualsExactType(TestAllTypes.NestedEnum.BAZ, message.getExtension(repeatedNestedEnumExtension, 1)); @@ -1664,6 +1720,7 @@ public final class TestUtil { Assert.assertEquals(0, message.getExtensionCount(repeatedNestedMessageExtension )); Assert.assertEquals(0, message.getExtensionCount(repeatedForeignMessageExtension)); Assert.assertEquals(0, message.getExtensionCount(repeatedImportMessageExtension )); + Assert.assertEquals(0, message.getExtensionCount(repeatedLazyMessageExtension )); Assert.assertEquals(0, message.getExtensionCount(repeatedNestedEnumExtension )); Assert.assertEquals(0, message.getExtensionCount(repeatedForeignEnumExtension )); Assert.assertEquals(0, message.getExtensionCount(repeatedImportEnumExtension )); @@ -1692,6 +1749,7 @@ public final class TestUtil { Assert.assertEquals(0, message.getExtension(repeatedNestedMessageExtension ).size()); Assert.assertEquals(0, message.getExtension(repeatedForeignMessageExtension).size()); Assert.assertEquals(0, message.getExtension(repeatedImportMessageExtension ).size()); + Assert.assertEquals(0, message.getExtension(repeatedLazyMessageExtension ).size()); Assert.assertEquals(0, message.getExtension(repeatedNestedEnumExtension ).size()); Assert.assertEquals(0, message.getExtension(repeatedForeignEnumExtension ).size()); Assert.assertEquals(0, message.getExtension(repeatedImportEnumExtension ).size()); @@ -1783,6 +1841,7 @@ public final class TestUtil { Assert.assertEquals(2, message.getExtensionCount(repeatedNestedMessageExtension )); Assert.assertEquals(2, message.getExtensionCount(repeatedForeignMessageExtension)); Assert.assertEquals(2, message.getExtensionCount(repeatedImportMessageExtension )); + Assert.assertEquals(2, message.getExtensionCount(repeatedLazyMessageExtension )); Assert.assertEquals(2, message.getExtensionCount(repeatedNestedEnumExtension )); Assert.assertEquals(2, message.getExtensionCount(repeatedForeignEnumExtension )); Assert.assertEquals(2, message.getExtensionCount(repeatedImportEnumExtension )); @@ -1810,6 +1869,7 @@ public final class TestUtil { assertEqualsExactType(218, message.getExtension(repeatedNestedMessageExtension , 0).getBb()); assertEqualsExactType(219, message.getExtension(repeatedForeignMessageExtension, 0).getC()); assertEqualsExactType(220, message.getExtension(repeatedImportMessageExtension , 0).getD()); + assertEqualsExactType(227, message.getExtension(repeatedLazyMessageExtension , 0).getBb()); assertEqualsExactType(TestAllTypes.NestedEnum.BAR, message.getExtension(repeatedNestedEnumExtension, 0)); @@ -1842,6 +1902,7 @@ public final class TestUtil { assertEqualsExactType(518, message.getExtension(repeatedNestedMessageExtension , 1).getBb()); assertEqualsExactType(519, message.getExtension(repeatedForeignMessageExtension, 1).getC()); assertEqualsExactType(520, message.getExtension(repeatedImportMessageExtension , 1).getD()); + assertEqualsExactType(527, message.getExtension(repeatedLazyMessageExtension , 1).getBb()); assertEqualsExactType(TestAllTypes.NestedEnum.FOO, message.getExtension(repeatedNestedEnumExtension, 1)); @@ -1965,6 +2026,10 @@ public final class TestUtil { ForeignMessageLite.newBuilder().setC(119).build()); message.setExtension(optionalImportMessageExtensionLite, ImportMessageLite.newBuilder().setD(120).build()); + message.setExtension(optionalPublicImportMessageExtensionLite, + PublicImportMessageLite.newBuilder().setE(126).build()); + message.setExtension(optionalLazyMessageExtensionLite, + TestAllTypesLite.NestedMessage.newBuilder().setBb(127).build()); message.setExtension(optionalNestedEnumExtensionLite, TestAllTypesLite.NestedEnum.BAZ); message.setExtension(optionalForeignEnumExtensionLite, ForeignEnumLite.FOREIGN_LITE_BAZ); @@ -1999,6 +2064,8 @@ public final class TestUtil { ForeignMessageLite.newBuilder().setC(219).build()); message.addExtension(repeatedImportMessageExtensionLite, ImportMessageLite.newBuilder().setD(220).build()); + message.addExtension(repeatedLazyMessageExtensionLite, + TestAllTypesLite.NestedMessage.newBuilder().setBb(227).build()); message.addExtension(repeatedNestedEnumExtensionLite, TestAllTypesLite.NestedEnum.BAR); message.addExtension(repeatedForeignEnumExtensionLite, ForeignEnumLite.FOREIGN_LITE_BAR); @@ -2032,6 +2099,8 @@ public final class TestUtil { ForeignMessageLite.newBuilder().setC(319).build()); message.addExtension(repeatedImportMessageExtensionLite, ImportMessageLite.newBuilder().setD(320).build()); + message.addExtension(repeatedLazyMessageExtensionLite, + TestAllTypesLite.NestedMessage.newBuilder().setBb(327).build()); message.addExtension(repeatedNestedEnumExtensionLite, TestAllTypesLite.NestedEnum.BAZ); message.addExtension(repeatedForeignEnumExtensionLite, ForeignEnumLite.FOREIGN_LITE_BAZ); @@ -2098,6 +2167,8 @@ public final class TestUtil { ForeignMessageLite.newBuilder().setC(519).build()); message.setExtension(repeatedImportMessageExtensionLite, 1, ImportMessageLite.newBuilder().setD(520).build()); + message.setExtension(repeatedLazyMessageExtensionLite, 1, + TestAllTypesLite.NestedMessage.newBuilder().setBb(527).build()); message.setExtension(repeatedNestedEnumExtensionLite , 1, TestAllTypesLite.NestedEnum.FOO); message.setExtension(repeatedForeignEnumExtensionLite, 1, ForeignEnumLite.FOREIGN_LITE_FOO); @@ -2168,6 +2239,9 @@ public final class TestUtil { assertEqualsExactType(118, message.getExtension(optionalNestedMessageExtensionLite ).getBb()); assertEqualsExactType(119, message.getExtension(optionalForeignMessageExtensionLite).getC()); assertEqualsExactType(120, message.getExtension(optionalImportMessageExtensionLite ).getD()); + assertEqualsExactType(126, message.getExtension( + optionalPublicImportMessageExtensionLite).getE()); + assertEqualsExactType(127, message.getExtension(optionalLazyMessageExtensionLite).getBb()); assertEqualsExactType(TestAllTypesLite.NestedEnum.BAZ, message.getExtension(optionalNestedEnumExtensionLite)); @@ -2201,6 +2275,7 @@ public final class TestUtil { Assert.assertEquals(2, message.getExtensionCount(repeatedNestedMessageExtensionLite )); Assert.assertEquals(2, message.getExtensionCount(repeatedForeignMessageExtensionLite)); Assert.assertEquals(2, message.getExtensionCount(repeatedImportMessageExtensionLite )); + Assert.assertEquals(2, message.getExtensionCount(repeatedLazyMessageExtensionLite )); Assert.assertEquals(2, message.getExtensionCount(repeatedNestedEnumExtensionLite )); Assert.assertEquals(2, message.getExtensionCount(repeatedForeignEnumExtensionLite )); Assert.assertEquals(2, message.getExtensionCount(repeatedImportEnumExtensionLite )); @@ -2228,6 +2303,7 @@ public final class TestUtil { assertEqualsExactType(218, message.getExtension(repeatedNestedMessageExtensionLite ,0).getBb()); assertEqualsExactType(219, message.getExtension(repeatedForeignMessageExtensionLite,0).getC()); assertEqualsExactType(220, message.getExtension(repeatedImportMessageExtensionLite ,0).getD()); + assertEqualsExactType(227, message.getExtension(repeatedLazyMessageExtensionLite ,0).getBb()); assertEqualsExactType(TestAllTypesLite.NestedEnum.BAR, message.getExtension(repeatedNestedEnumExtensionLite, 0)); @@ -2259,6 +2335,7 @@ public final class TestUtil { assertEqualsExactType(318, message.getExtension(repeatedNestedMessageExtensionLite ,1).getBb()); assertEqualsExactType(319, message.getExtension(repeatedForeignMessageExtensionLite,1).getC()); assertEqualsExactType(320, message.getExtension(repeatedImportMessageExtensionLite ,1).getD()); + assertEqualsExactType(327, message.getExtension(repeatedLazyMessageExtensionLite ,1).getBb()); assertEqualsExactType(TestAllTypesLite.NestedEnum.BAZ, message.getExtension(repeatedNestedEnumExtensionLite, 1)); @@ -2348,10 +2425,12 @@ public final class TestUtil { Assert.assertFalse(message.hasExtension(optionalStringExtensionLite )); Assert.assertFalse(message.hasExtension(optionalBytesExtensionLite )); - Assert.assertFalse(message.hasExtension(optionalGroupExtensionLite )); - Assert.assertFalse(message.hasExtension(optionalNestedMessageExtensionLite )); - Assert.assertFalse(message.hasExtension(optionalForeignMessageExtensionLite)); - Assert.assertFalse(message.hasExtension(optionalImportMessageExtensionLite )); + Assert.assertFalse(message.hasExtension(optionalGroupExtensionLite )); + Assert.assertFalse(message.hasExtension(optionalNestedMessageExtensionLite )); + Assert.assertFalse(message.hasExtension(optionalForeignMessageExtensionLite )); + Assert.assertFalse(message.hasExtension(optionalImportMessageExtensionLite )); + Assert.assertFalse(message.hasExtension(optionalPublicImportMessageExtensionLite)); + Assert.assertFalse(message.hasExtension(optionalLazyMessageExtensionLite )); Assert.assertFalse(message.hasExtension(optionalNestedEnumExtensionLite )); Assert.assertFalse(message.hasExtension(optionalForeignEnumExtensionLite)); @@ -2378,15 +2457,20 @@ public final class TestUtil { assertEqualsExactType(ByteString.EMPTY, message.getExtension(optionalBytesExtensionLite)); // Embedded messages should also be clear. - Assert.assertFalse(message.getExtension(optionalGroupExtensionLite ).hasA()); - Assert.assertFalse(message.getExtension(optionalNestedMessageExtensionLite ).hasBb()); - Assert.assertFalse(message.getExtension(optionalForeignMessageExtensionLite).hasC()); - Assert.assertFalse(message.getExtension(optionalImportMessageExtensionLite ).hasD()); + Assert.assertFalse(message.getExtension(optionalGroupExtensionLite ).hasA()); + Assert.assertFalse(message.getExtension(optionalNestedMessageExtensionLite ).hasBb()); + Assert.assertFalse(message.getExtension(optionalForeignMessageExtensionLite ).hasC()); + Assert.assertFalse(message.getExtension(optionalImportMessageExtensionLite ).hasD()); + Assert.assertFalse(message.getExtension(optionalPublicImportMessageExtensionLite).hasE()); + Assert.assertFalse(message.getExtension(optionalLazyMessageExtensionLite ).hasBb()); assertEqualsExactType(0, message.getExtension(optionalGroupExtensionLite ).getA()); assertEqualsExactType(0, message.getExtension(optionalNestedMessageExtensionLite ).getBb()); assertEqualsExactType(0, message.getExtension(optionalForeignMessageExtensionLite).getC()); assertEqualsExactType(0, message.getExtension(optionalImportMessageExtensionLite ).getD()); + assertEqualsExactType(0, message.getExtension( + optionalPublicImportMessageExtensionLite).getE()); + assertEqualsExactType(0, message.getExtension(optionalLazyMessageExtensionLite ).getBb()); // Enums without defaults are set to the first value in the enum. assertEqualsExactType(TestAllTypesLite.NestedEnum.FOO, @@ -2420,6 +2504,7 @@ public final class TestUtil { Assert.assertEquals(0, message.getExtensionCount(repeatedNestedMessageExtensionLite )); Assert.assertEquals(0, message.getExtensionCount(repeatedForeignMessageExtensionLite)); Assert.assertEquals(0, message.getExtensionCount(repeatedImportMessageExtensionLite )); + Assert.assertEquals(0, message.getExtensionCount(repeatedLazyMessageExtensionLite )); Assert.assertEquals(0, message.getExtensionCount(repeatedNestedEnumExtensionLite )); Assert.assertEquals(0, message.getExtensionCount(repeatedForeignEnumExtensionLite )); Assert.assertEquals(0, message.getExtensionCount(repeatedImportEnumExtensionLite )); @@ -2511,6 +2596,7 @@ public final class TestUtil { Assert.assertEquals(2, message.getExtensionCount(repeatedNestedMessageExtensionLite )); Assert.assertEquals(2, message.getExtensionCount(repeatedForeignMessageExtensionLite)); Assert.assertEquals(2, message.getExtensionCount(repeatedImportMessageExtensionLite )); + Assert.assertEquals(2, message.getExtensionCount(repeatedLazyMessageExtensionLite )); Assert.assertEquals(2, message.getExtensionCount(repeatedNestedEnumExtensionLite )); Assert.assertEquals(2, message.getExtensionCount(repeatedForeignEnumExtensionLite )); Assert.assertEquals(2, message.getExtensionCount(repeatedImportEnumExtensionLite )); @@ -2538,6 +2624,7 @@ public final class TestUtil { assertEqualsExactType(218, message.getExtension(repeatedNestedMessageExtensionLite ,0).getBb()); assertEqualsExactType(219, message.getExtension(repeatedForeignMessageExtensionLite,0).getC()); assertEqualsExactType(220, message.getExtension(repeatedImportMessageExtensionLite ,0).getD()); + assertEqualsExactType(227, message.getExtension(repeatedLazyMessageExtensionLite ,0).getBb()); assertEqualsExactType(TestAllTypesLite.NestedEnum.BAR, message.getExtension(repeatedNestedEnumExtensionLite, 0)); @@ -2570,6 +2657,7 @@ public final class TestUtil { assertEqualsExactType(518, message.getExtension(repeatedNestedMessageExtensionLite ,1).getBb()); assertEqualsExactType(519, message.getExtension(repeatedForeignMessageExtensionLite,1).getC()); assertEqualsExactType(520, message.getExtension(repeatedImportMessageExtensionLite ,1).getD()); + assertEqualsExactType(527, message.getExtension(repeatedLazyMessageExtensionLite ,1).getBb()); assertEqualsExactType(TestAllTypesLite.NestedEnum.FOO, message.getExtension(repeatedNestedEnumExtensionLite, 1)); @@ -2674,18 +2762,21 @@ public final class TestUtil { private final Descriptors.FileDescriptor file; private final Descriptors.FileDescriptor importFile; + private final Descriptors.FileDescriptor publicImportFile; private final Descriptors.Descriptor optionalGroup; private final Descriptors.Descriptor repeatedGroup; private final Descriptors.Descriptor nestedMessage; private final Descriptors.Descriptor foreignMessage; private final Descriptors.Descriptor importMessage; + private final Descriptors.Descriptor publicImportMessage; private final Descriptors.FieldDescriptor groupA; private final Descriptors.FieldDescriptor repeatedGroupA; private final Descriptors.FieldDescriptor nestedB; private final Descriptors.FieldDescriptor foreignC; private final Descriptors.FieldDescriptor importD; + private final Descriptors.FieldDescriptor importE; private final Descriptors.EnumDescriptor nestedEnum; private final Descriptors.EnumDescriptor foreignEnum; @@ -2722,6 +2813,7 @@ public final class TestUtil { this.file = baseDescriptor.getFile(); Assert.assertEquals(1, file.getDependencies().size()); this.importFile = file.getDependencies().get(0); + this.publicImportFile = importFile.getDependencies().get(0); Descriptors.Descriptor testAllTypes; if (baseDescriptor.getName() == "TestAllTypes") { @@ -2748,6 +2840,8 @@ public final class TestUtil { this.nestedMessage = testAllTypes.findNestedTypeByName("NestedMessage"); this.foreignMessage = file.findMessageTypeByName("ForeignMessage"); this.importMessage = importFile.findMessageTypeByName("ImportMessage"); + this.publicImportMessage = publicImportFile.findMessageTypeByName( + "PublicImportMessage"); this.nestedEnum = testAllTypes.findEnumTypeByName("NestedEnum"); this.foreignEnum = file.findEnumTypeByName("ForeignEnum"); @@ -2765,6 +2859,7 @@ public final class TestUtil { this.nestedB = nestedMessage .findFieldByName("bb"); this.foreignC = foreignMessage.findFieldByName("c"); this.importD = importMessage .findFieldByName("d"); + this.importE = publicImportMessage.findFieldByName("e"); this.nestedFoo = nestedEnum.findValueByName("FOO"); this.nestedBar = nestedEnum.findValueByName("BAR"); this.nestedBaz = nestedEnum.findValueByName("BAZ"); @@ -2783,6 +2878,7 @@ public final class TestUtil { Assert.assertNotNull(nestedB ); Assert.assertNotNull(foreignC ); Assert.assertNotNull(importD ); + Assert.assertNotNull(importE ); Assert.assertNotNull(nestedFoo ); Assert.assertNotNull(nestedBar ); Assert.assertNotNull(nestedBaz ); @@ -2863,6 +2959,12 @@ public final class TestUtil { message.setField(f("optional_import_message"), newBuilderForField(message, f("optional_import_message")) .setField(importD, 120).build()); + message.setField(f("optional_public_import_message"), + newBuilderForField(message, f("optional_public_import_message")) + .setField(importE, 126).build()); + message.setField(f("optional_lazy_message"), + newBuilderForField(message, f("optional_lazy_message")) + .setField(nestedB, 127).build()); message.setField(f("optional_nested_enum" ), nestedBaz); message.setField(f("optional_foreign_enum"), foreignBaz); @@ -2901,6 +3003,9 @@ public final class TestUtil { message.addRepeatedField(f("repeated_import_message"), newBuilderForField(message, f("repeated_import_message")) .setField(importD, 220).build()); + message.addRepeatedField(f("repeated_lazy_message"), + newBuilderForField(message, f("repeated_lazy_message")) + .setField(nestedB, 227).build()); message.addRepeatedField(f("repeated_nested_enum" ), nestedBar); message.addRepeatedField(f("repeated_foreign_enum"), foreignBar); @@ -2938,6 +3043,9 @@ public final class TestUtil { message.addRepeatedField(f("repeated_import_message"), newBuilderForField(message, f("repeated_import_message")) .setField(importD, 320).build()); + message.addRepeatedField(f("repeated_lazy_message"), + newBuilderForField(message, f("repeated_lazy_message")) + .setField(nestedB, 327).build()); message.addRepeatedField(f("repeated_nested_enum" ), nestedBaz); message.addRepeatedField(f("repeated_foreign_enum"), foreignBaz); @@ -3008,6 +3116,9 @@ public final class TestUtil { message.setRepeatedField(f("repeated_import_message"), 1, newBuilderForField(message, f("repeated_import_message")) .setField(importD, 520).build()); + message.setRepeatedField(f("repeated_lazy_message"), 1, + newBuilderForField(message, f("repeated_lazy_message")) + .setField(nestedB, 527).build()); message.setRepeatedField(f("repeated_nested_enum" ), 1, nestedFoo); message.setRepeatedField(f("repeated_foreign_enum"), 1, foreignFoo); @@ -3092,6 +3203,12 @@ public final class TestUtil { Assert.assertEquals(120, ((Message)message.getField(f("optional_import_message"))) .getField(importD)); + Assert.assertEquals(126, + ((Message)message.getField(f("optional_public_import_message"))) + .getField(importE)); + Assert.assertEquals(127, + ((Message)message.getField(f("optional_lazy_message"))) + .getField(nestedB)); Assert.assertEquals( nestedBaz, message.getField(f("optional_nested_enum" ))); Assert.assertEquals(foreignBaz, message.getField(f("optional_foreign_enum"))); @@ -3122,6 +3239,7 @@ public final class TestUtil { Assert.assertEquals(2, message.getRepeatedFieldCount(f("repeated_nested_message" ))); Assert.assertEquals(2, message.getRepeatedFieldCount(f("repeated_foreign_message"))); Assert.assertEquals(2, message.getRepeatedFieldCount(f("repeated_import_message" ))); + Assert.assertEquals(2, message.getRepeatedFieldCount(f("repeated_lazy_message" ))); Assert.assertEquals(2, message.getRepeatedFieldCount(f("repeated_nested_enum" ))); Assert.assertEquals(2, message.getRepeatedFieldCount(f("repeated_foreign_enum" ))); Assert.assertEquals(2, message.getRepeatedFieldCount(f("repeated_import_enum" ))); @@ -3157,6 +3275,9 @@ public final class TestUtil { Assert.assertEquals(220, ((Message)message.getRepeatedField(f("repeated_import_message"), 0)) .getField(importD)); + Assert.assertEquals(227, + ((Message)message.getRepeatedField(f("repeated_lazy_message"), 0)) + .getField(nestedB)); Assert.assertEquals( nestedBar, message.getRepeatedField(f("repeated_nested_enum" ),0)); Assert.assertEquals(foreignBar, message.getRepeatedField(f("repeated_foreign_enum"),0)); @@ -3193,6 +3314,9 @@ public final class TestUtil { Assert.assertEquals(320, ((Message)message.getRepeatedField(f("repeated_import_message"), 1)) .getField(importD)); + Assert.assertEquals(327, + ((Message)message.getRepeatedField(f("repeated_lazy_message"), 1)) + .getField(nestedB)); Assert.assertEquals( nestedBaz, message.getRepeatedField(f("repeated_nested_enum" ),1)); Assert.assertEquals(foreignBaz, message.getRepeatedField(f("repeated_foreign_enum"),1)); @@ -3316,6 +3440,12 @@ public final class TestUtil { Assert.assertFalse( ((Message)message.getField(f("optional_import_message"))) .hasField(importD)); + Assert.assertFalse( + ((Message)message.getField(f("optional_public_import_message"))) + .hasField(importE)); + Assert.assertFalse( + ((Message)message.getField(f("optional_lazy_message"))) + .hasField(nestedB)); Assert.assertEquals(0, ((Message)message.getField(f("optionalgroup"))).getField(groupA)); @@ -3328,6 +3458,12 @@ public final class TestUtil { Assert.assertEquals(0, ((Message)message.getField(f("optional_import_message"))) .getField(importD)); + Assert.assertEquals(0, + ((Message)message.getField(f("optional_public_import_message"))) + .getField(importE)); + Assert.assertEquals(0, + ((Message)message.getField(f("optional_lazy_message"))) + .getField(nestedB)); // Enums without defaults are set to the first value in the enum. Assert.assertEquals( nestedFoo, message.getField(f("optional_nested_enum" ))); @@ -3358,6 +3494,7 @@ public final class TestUtil { Assert.assertEquals(0, message.getRepeatedFieldCount(f("repeated_nested_message" ))); Assert.assertEquals(0, message.getRepeatedFieldCount(f("repeated_foreign_message"))); Assert.assertEquals(0, message.getRepeatedFieldCount(f("repeated_import_message" ))); + Assert.assertEquals(0, message.getRepeatedFieldCount(f("repeated_lazy_message" ))); Assert.assertEquals(0, message.getRepeatedFieldCount(f("repeated_nested_enum" ))); Assert.assertEquals(0, message.getRepeatedFieldCount(f("repeated_foreign_enum" ))); Assert.assertEquals(0, message.getRepeatedFieldCount(f("repeated_import_enum" ))); @@ -3442,6 +3579,7 @@ public final class TestUtil { Assert.assertEquals(2, message.getRepeatedFieldCount(f("repeated_nested_message" ))); Assert.assertEquals(2, message.getRepeatedFieldCount(f("repeated_foreign_message"))); Assert.assertEquals(2, message.getRepeatedFieldCount(f("repeated_import_message" ))); + Assert.assertEquals(2, message.getRepeatedFieldCount(f("repeated_lazy_message" ))); Assert.assertEquals(2, message.getRepeatedFieldCount(f("repeated_nested_enum" ))); Assert.assertEquals(2, message.getRepeatedFieldCount(f("repeated_foreign_enum" ))); Assert.assertEquals(2, message.getRepeatedFieldCount(f("repeated_import_enum" ))); @@ -3477,6 +3615,9 @@ public final class TestUtil { Assert.assertEquals(220, ((Message)message.getRepeatedField(f("repeated_import_message"), 0)) .getField(importD)); + Assert.assertEquals(227, + ((Message)message.getRepeatedField(f("repeated_lazy_message"), 0)) + .getField(nestedB)); Assert.assertEquals( nestedBar, message.getRepeatedField(f("repeated_nested_enum" ),0)); Assert.assertEquals(foreignBar, message.getRepeatedField(f("repeated_foreign_enum"),0)); @@ -3513,6 +3654,9 @@ public final class TestUtil { Assert.assertEquals(520, ((Message)message.getRepeatedField(f("repeated_import_message"), 1)) .getField(importD)); + Assert.assertEquals(527, + ((Message)message.getRepeatedField(f("repeated_lazy_message"), 1)) + .getField(nestedB)); Assert.assertEquals( nestedFoo, message.getRepeatedField(f("repeated_nested_enum" ),1)); Assert.assertEquals(foreignFoo, message.getRepeatedField(f("repeated_foreign_enum"),1)); diff --git a/java/src/test/java/com/google/protobuf/TextFormatTest.java b/java/src/test/java/com/google/protobuf/TextFormatTest.java index 47690d1f..5323d70e 100644 --- a/java/src/test/java/com/google/protobuf/TextFormatTest.java +++ b/java/src/test/java/com/google/protobuf/TextFormatTest.java @@ -121,6 +121,18 @@ public class TextFormatTest extends TestCase { assertEquals(allFieldsSetText, javaText); } + /** Print TestAllTypes as Builder and compare with golden file. */ + public void testPrintMessageBuilder() throws Exception { + String javaText = TextFormat.printToString(TestUtil.getAllSetBuilder()); + + // Java likes to add a trailing ".0" to floats and doubles. C printf + // (with %g format) does not. Our golden files are used for both + // C++ and Java TextFormat classes, so we need to conform. + javaText = javaText.replace(".0\n", "\n"); + + assertEquals(allFieldsSetText, javaText); + } + /** Print TestAllExtensions and compare with golden file. */ public void testPrintExtensions() throws Exception { String javaText = TextFormat.printToString(TestUtil.getAllExtensionsSet()); @@ -749,4 +761,26 @@ public class TextFormatTest extends TestCase { + " 0xabcdef1234567890", TextFormat.shortDebugString(makeUnknownFieldSet())); } + + public void testPrintToUnicodeString() { + assertEquals( + "optional_string: \"abc\u3042efg\"\n" + + "optional_bytes: \"\\343\\201\\202\"\n" + + "repeated_string: \"\u3093XYZ\"\n", + TextFormat.printToUnicodeString(TestAllTypes.newBuilder() + .setOptionalString("abc\u3042efg") + .setOptionalBytes(bytes(0xe3, 0x81, 0x82)) + .addRepeatedString("\u3093XYZ") + .build())); + } + + public void testPrintToUnicodeString_unknown() { + assertEquals( + "1: \"\\343\\201\\202\"\n", + TextFormat.printToUnicodeString(UnknownFieldSet.newBuilder() + .addField(1, + UnknownFieldSet.Field.newBuilder() + .addLengthDelimited(bytes(0xe3, 0x81, 0x82)).build()) + .build())); + } } diff --git a/java/src/test/java/com/google/protobuf/WireFormatTest.java b/java/src/test/java/com/google/protobuf/WireFormatTest.java index 5ea1dd6a..94f62134 100644 --- a/java/src/test/java/com/google/protobuf/WireFormatTest.java +++ b/java/src/test/java/com/google/protobuf/WireFormatTest.java @@ -34,6 +34,7 @@ import junit.framework.TestCase; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.util.List; import protobuf_unittest.UnittestProto; import protobuf_unittest.UnittestProto.TestAllExtensions; @@ -328,7 +329,17 @@ public class WireFormatTest extends TestCase { private static final int TYPE_ID_2 = TestMessageSetExtension2.getDescriptor().getExtensions().get(0).getNumber(); - public void testSerializeMessageSet() throws Exception { + public void testSerializeMessageSetEagerly() throws Exception { + testSerializeMessageSetWithFlag(true); + } + + public void testSerializeMessageSetNotEagerly() throws Exception { + testSerializeMessageSetWithFlag(false); + } + + private void testSerializeMessageSetWithFlag(boolean eagerParsing) + throws Exception { + ExtensionRegistryLite.setEagerlyParseMessageSets(eagerParsing); // Set up a TestMessageSet with two known messages and an unknown one. TestMessageSet messageSet = TestMessageSet.newBuilder() @@ -372,7 +383,17 @@ public class WireFormatTest extends TestCase { assertEquals("bar", raw.getItem(2).getMessage().toStringUtf8()); } - public void testParseMessageSet() throws Exception { + public void testParseMessageSetEagerly() throws Exception { + testParseMessageSetWithFlag(true); + } + + public void testParseMessageSetNotEagerly()throws Exception { + testParseMessageSetWithFlag(false); + } + + private void testParseMessageSetWithFlag(boolean eagerParsing) + throws Exception { + ExtensionRegistryLite.setEagerlyParseMessageSets(eagerParsing); ExtensionRegistry extensionRegistry = ExtensionRegistry.newInstance(); extensionRegistry.add(TestMessageSetExtension1.messageSetExtension); extensionRegistry.add(TestMessageSetExtension2.messageSetExtension); @@ -424,4 +445,136 @@ public class WireFormatTest extends TestCase { assertEquals(1, field.getLengthDelimitedList().size()); assertEquals("bar", field.getLengthDelimitedList().get(0).toStringUtf8()); } + + public void testParseMessageSetExtensionEagerly() throws Exception { + testParseMessageSetExtensionWithFlag(true); + } + + public void testParseMessageSetExtensionNotEagerly() throws Exception { + testParseMessageSetExtensionWithFlag(false); + } + + private void testParseMessageSetExtensionWithFlag(boolean eagerParsing) + throws Exception { + ExtensionRegistryLite.setEagerlyParseMessageSets(eagerParsing); + ExtensionRegistry extensionRegistry = ExtensionRegistry.newInstance(); + extensionRegistry.add(TestMessageSetExtension1.messageSetExtension); + + // Set up a RawMessageSet with a known messages. + int TYPE_ID_1 = + TestMessageSetExtension1 + .getDescriptor().getExtensions().get(0).getNumber(); + RawMessageSet raw = + RawMessageSet.newBuilder() + .addItem( + RawMessageSet.Item.newBuilder() + .setTypeId(TYPE_ID_1) + .setMessage( + TestMessageSetExtension1.newBuilder() + .setI(123) + .build().toByteString()) + .build()) + .build(); + + ByteString data = raw.toByteString(); + + // Parse as a TestMessageSet and check the contents. + TestMessageSet messageSet = + TestMessageSet.parseFrom(data, extensionRegistry); + assertEquals(123, messageSet.getExtension( + TestMessageSetExtension1.messageSetExtension).getI()); + } + + public void testMergeLazyMessageSetExtensionEagerly() throws Exception { + testMergeLazyMessageSetExtensionWithFlag(true); + } + + public void testMergeLazyMessageSetExtensionNotEagerly() throws Exception { + testMergeLazyMessageSetExtensionWithFlag(false); + } + + private void testMergeLazyMessageSetExtensionWithFlag(boolean eagerParsing) + throws Exception { + ExtensionRegistryLite.setEagerlyParseMessageSets(eagerParsing); + ExtensionRegistry extensionRegistry = ExtensionRegistry.newInstance(); + extensionRegistry.add(TestMessageSetExtension1.messageSetExtension); + + // Set up a RawMessageSet with a known messages. + int TYPE_ID_1 = + TestMessageSetExtension1 + .getDescriptor().getExtensions().get(0).getNumber(); + RawMessageSet raw = + RawMessageSet.newBuilder() + .addItem( + RawMessageSet.Item.newBuilder() + .setTypeId(TYPE_ID_1) + .setMessage( + TestMessageSetExtension1.newBuilder() + .setI(123) + .build().toByteString()) + .build()) + .build(); + + ByteString data = raw.toByteString(); + + // Parse as a TestMessageSet and store value into lazy field + TestMessageSet messageSet = + TestMessageSet.parseFrom(data, extensionRegistry); + // Merge lazy field check the contents. + messageSet = + messageSet.toBuilder().mergeFrom(data, extensionRegistry).build(); + assertEquals(123, messageSet.getExtension( + TestMessageSetExtension1.messageSetExtension).getI()); + } + + public void testMergeMessageSetExtensionEagerly() throws Exception { + testMergeMessageSetExtensionWithFlag(true); + } + + public void testMergeMessageSetExtensionNotEagerly() throws Exception { + testMergeMessageSetExtensionWithFlag(false); + } + + private void testMergeMessageSetExtensionWithFlag(boolean eagerParsing) + throws Exception { + ExtensionRegistryLite.setEagerlyParseMessageSets(eagerParsing); + ExtensionRegistry extensionRegistry = ExtensionRegistry.newInstance(); + extensionRegistry.add(TestMessageSetExtension1.messageSetExtension); + + // Set up a RawMessageSet with a known messages. + int TYPE_ID_1 = + TestMessageSetExtension1 + .getDescriptor().getExtensions().get(0).getNumber(); + RawMessageSet raw = + RawMessageSet.newBuilder() + .addItem( + RawMessageSet.Item.newBuilder() + .setTypeId(TYPE_ID_1) + .setMessage( + TestMessageSetExtension1.newBuilder() + .setI(123) + .build().toByteString()) + .build()) + .build(); + + // Serialize RawMessageSet unnormally (message value before type id) + ByteString.CodedBuilder out = ByteString.newCodedBuilder( + raw.getSerializedSize()); + CodedOutputStream output = out.getCodedOutput(); + List items = raw.getItemList(); + for (int i = 0; i < items.size(); i++) { + RawMessageSet.Item item = items.get(i); + output.writeTag(1, WireFormat.WIRETYPE_START_GROUP); + output.writeBytes(3, item.getMessage()); + output.writeInt32(2, item.getTypeId()); + output.writeTag(1, WireFormat.WIRETYPE_END_GROUP); + } + ByteString data = out.build(); + + // Merge bytes into TestMessageSet and check the contents. + TestMessageSet messageSet = + TestMessageSet.newBuilder().mergeFrom(data, extensionRegistry).build(); + assertEquals(123, messageSet.getExtension( + TestMessageSetExtension1.messageSetExtension).getI()); + } } diff --git a/java/src/test/java/com/google/protobuf/test_bad_identifiers.proto b/java/src/test/java/com/google/protobuf/test_bad_identifiers.proto index 9610e9d5..6e67d97a 100644 --- a/java/src/test/java/com/google/protobuf/test_bad_identifiers.proto +++ b/java/src/test/java/com/google/protobuf/test_bad_identifiers.proto @@ -46,6 +46,23 @@ option java_outer_classname = "TestBadIdentifiersProto"; message TestMessage { } +message Descriptor { + option no_standard_descriptor_accessor = true; + optional string descriptor = 1; + message NestedDescriptor { + option no_standard_descriptor_accessor = true; + optional string descriptor = 1; + } + optional NestedDescriptor nested_descriptor = 2; +} + +message Parser { + enum ParserEnum { + PARSER = 1; + } + optional ParserEnum parser = 1; +} + message Deprecated { enum TestEnum { FOO = 1; @@ -62,6 +79,7 @@ message Override { message Object { optional int32 object = 1; + optional string string_object = 2; } message String { -- cgit v1.2.3