diff options
author | Bo Yang <teboring@google.com> | 2016-09-19 13:45:07 -0700 |
---|---|---|
committer | Bo Yang <teboring@google.com> | 2016-09-19 17:42:40 -0700 |
commit | 98835fb8f8a8f144f2a1eac0b833ff83f4a05f54 (patch) | |
tree | 49504df5c2020eb085c1826548c10a3cbd188e12 /java/core | |
parent | b5bbdb0967109fbae7bfc423ef3cd6dc776359b6 (diff) | |
download | protobuf-98835fb8f8a8f144f2a1eac0b833ff83f4a05f54.tar.gz protobuf-98835fb8f8a8f144f2a1eac0b833ff83f4a05f54.tar.bz2 protobuf-98835fb8f8a8f144f2a1eac0b833ff83f4a05f54.zip |
Integrate internal changes
Diffstat (limited to 'java/core')
60 files changed, 4512 insertions, 1544 deletions
diff --git a/java/core/src/main/java/com/google/protobuf/ByteString.java b/java/core/src/main/java/com/google/protobuf/ByteString.java index 3f3f9f3c..5b24976d 100644 --- a/java/core/src/main/java/com/google/protobuf/ByteString.java +++ b/java/core/src/main/java/com/google/protobuf/ByteString.java @@ -311,6 +311,18 @@ public abstract class ByteString implements Iterable<Byte>, Serializable { } /** + * Wraps the given bytes into a {@code ByteString}. Intended for internal only usage. + */ + static ByteString wrap(ByteBuffer buffer) { + if (buffer.hasArray()) { + final int offset = buffer.arrayOffset(); + return ByteString.wrap(buffer.array(), offset + buffer.position(), buffer.remaining()); + } else { + return new NioByteString(buffer); + } + } + + /** * Wraps the given bytes into a {@code ByteString}. Intended for internal only * usage to force a classload of ByteString before LiteralByteString. */ @@ -679,6 +691,7 @@ public abstract class ByteString implements Iterable<Byte>, Serializable { */ abstract void writeTo(ByteOutput byteOutput) throws IOException; + /** * Constructs a read-only {@code java.nio.ByteBuffer} whose content * is equal to the contents of this byte string. @@ -820,6 +833,7 @@ public abstract class ByteString implements Iterable<Byte>, Serializable { return true; } + /** * Check equality of the substring of given length of this object starting at * zero with another {@code ByteString} substring starting at offset. diff --git a/java/core/src/main/java/com/google/protobuf/CodedInputStream.java b/java/core/src/main/java/com/google/protobuf/CodedInputStream.java index e8860651..e461fa28 100644 --- a/java/core/src/main/java/com/google/protobuf/CodedInputStream.java +++ b/java/core/src/main/java/com/google/protobuf/CodedInputStream.java @@ -30,6 +30,14 @@ package com.google.protobuf; +import static com.google.protobuf.Internal.EMPTY_BYTE_ARRAY; +import static com.google.protobuf.Internal.EMPTY_BYTE_BUFFER; +import static com.google.protobuf.Internal.UTF_8; +import static com.google.protobuf.Internal.checkNotNull; +import static com.google.protobuf.WireFormat.FIXED_32_SIZE; +import static com.google.protobuf.WireFormat.FIXED_64_SIZE; +import static com.google.protobuf.WireFormat.MAX_VARINT_SIZE; + import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; @@ -41,51 +49,55 @@ import java.util.List; /** * Reads and decodes protocol message fields. * - * This class contains two kinds of methods: methods that read specific - * protocol message constructs and field types (e.g. {@link #readTag()} and - * {@link #readInt32()}) and methods that read low-level values (e.g. - * {@link #readRawVarint32()} and {@link #readRawBytes}). If you are reading - * encoded protocol messages, you should use the former methods, but if you are - * reading some other format of your own design, use the latter. + * <p>This class contains two kinds of methods: methods that read specific protocol message + * constructs and field types (e.g. {@link #readTag()} and {@link #readInt32()}) and methods that + * read low-level values (e.g. {@link #readRawVarint32()} and {@link #readRawBytes}). If you are + * reading encoded protocol messages, you should use the former methods, but if you are reading some + * other format of your own design, use the latter. * * @author kenton@google.com Kenton Varda */ -public final class CodedInputStream { - /** - * Create a new CodedInputStream wrapping the given InputStream. - */ +public abstract class CodedInputStream { + private static final int DEFAULT_BUFFER_SIZE = 4096; + private static final int DEFAULT_RECURSION_LIMIT = 100; + private static final int DEFAULT_SIZE_LIMIT = 64 << 20; // 64MB + + /** Visible for subclasses. See setRecursionLimit() */ + int recursionDepth; + + int recursionLimit = DEFAULT_RECURSION_LIMIT; + + /** Visible for subclasses. See setSizeLimit() */ + int sizeLimit = DEFAULT_SIZE_LIMIT; + + /** Create a new CodedInputStream wrapping the given InputStream. */ public static CodedInputStream newInstance(final InputStream input) { - return new CodedInputStream(input, BUFFER_SIZE); + return newInstance(input, DEFAULT_BUFFER_SIZE); } - - /** - * Create a new CodedInputStream wrapping the given InputStream. - */ + + /** Create a new CodedInputStream wrapping the given InputStream. */ static CodedInputStream newInstance(final InputStream input, int bufferSize) { - return new CodedInputStream(input, bufferSize); + if (input == null) { + // TODO(nathanmittler): Ideally we should throw here. This is done for backward compatibility. + return newInstance(EMPTY_BYTE_ARRAY); + } + return new StreamDecoder(input, bufferSize); } - /** - * Create a new CodedInputStream wrapping the given byte array. - */ + /** Create a new CodedInputStream wrapping the given byte array. */ public static CodedInputStream newInstance(final byte[] buf) { return newInstance(buf, 0, buf.length); } - /** - * Create a new CodedInputStream wrapping the given byte array slice. - */ - public static CodedInputStream newInstance(final byte[] buf, final int off, - final int len) { + /** Create a new CodedInputStream wrapping the given byte array slice. */ + public static CodedInputStream newInstance(final byte[] buf, final int off, final int len) { return newInstance(buf, off, len, false /* bufferIsImmutable */); } - - /** - * Create a new CodedInputStream wrapping the given byte array slice. - */ + + /** Create a new CodedInputStream wrapping the given byte array slice. */ static CodedInputStream newInstance( final byte[] buf, final int off, final int len, final boolean bufferIsImmutable) { - CodedInputStream result = new CodedInputStream(buf, off, len, bufferIsImmutable); + ArrayDecoder result = new ArrayDecoder(buf, off, len, bufferIsImmutable); try { // Some uses of CodedInputStream can be more efficient if they know // exactly how many bytes are available. By pushing the end point of the @@ -107,583 +119,358 @@ public final class CodedInputStream { } /** - * Create a new CodedInputStream wrapping the given ByteBuffer. The data - * starting from the ByteBuffer's current position to its limit will be read. - * The returned CodedInputStream may or may not share the underlying data - * in the ByteBuffer, therefore the ByteBuffer cannot be changed while the - * CodedInputStream is in use. - * Note that the ByteBuffer's position won't be changed by this function. - * Concurrent calls with the same ByteBuffer object are safe if no other - * thread is trying to alter the ByteBuffer's status. + * Create a new CodedInputStream wrapping the given ByteBuffer. The data starting from the + * ByteBuffer's current position to its limit will be read. The returned CodedInputStream may or + * may not share the underlying data in the ByteBuffer, therefore the ByteBuffer cannot be changed + * while the CodedInputStream is in use. Note that the ByteBuffer's position won't be changed by + * this function. Concurrent calls with the same ByteBuffer object are safe if no other thread is + * trying to alter the ByteBuffer's status. */ public static CodedInputStream newInstance(ByteBuffer buf) { + return newInstance(buf, false /* bufferIsImmutable */); + } + + /** Create a new CodedInputStream wrapping the given buffer. */ + static CodedInputStream newInstance(ByteBuffer buf, boolean bufferIsImmutable) { if (buf.hasArray()) { - return newInstance(buf.array(), buf.arrayOffset() + buf.position(), - buf.remaining()); - } else { - ByteBuffer temp = buf.duplicate(); - byte[] buffer = new byte[temp.remaining()]; - temp.get(buffer); - return newInstance(buffer); + return newInstance( + buf.array(), buf.arrayOffset() + buf.position(), buf.remaining(), bufferIsImmutable); + } + + if (buf.isDirect() && UnsafeDirectNioDecoder.isSupported()) { + return new UnsafeDirectNioDecoder(buf, bufferIsImmutable); } + + // The buffer is non-direct and does not expose the underlying array. Using the ByteBuffer API + // to access individual bytes is very slow, so just copy the buffer to an array. + // TODO(nathanmittler): Re-evaluate with Java 9 + byte[] buffer = new byte[buf.remaining()]; + buf.duplicate().get(buffer); + return newInstance(buffer, 0, buffer.length, true); } + /** Disable construction/inheritance outside of this class. */ + private CodedInputStream() {} + // ----------------------------------------------------------------- /** - * Attempt to read a field tag, returning zero if we have reached EOF. - * Protocol message parsers use this to read tags, since a protocol message - * may legally end wherever a tag occurs, and zero is not a valid tag number. + * Attempt to read a field tag, returning zero if we have reached EOF. Protocol message parsers + * use this to read tags, since a protocol message may legally end wherever a tag occurs, and zero + * is not a valid tag number. */ - public int readTag() throws IOException { - if (isAtEnd()) { - lastTag = 0; - return 0; - } - - lastTag = readRawVarint32(); - if (WireFormat.getTagFieldNumber(lastTag) == 0) { - // If we actually read zero (or any tag number corresponding to field - // number zero), that's not a valid tag. - throw InvalidProtocolBufferException.invalidTag(); - } - return lastTag; - } + public abstract int readTag() throws IOException; /** - * Verifies that the last call to readTag() returned the given tag value. - * This is used to verify that a nested group ended with the correct - * end tag. + * Verifies that the last call to readTag() returned the given tag value. This is used to verify + * that a nested group ended with the correct end tag. * - * @throws InvalidProtocolBufferException {@code value} does not match the - * last tag. + * @throws InvalidProtocolBufferException {@code value} does not match the last tag. */ - public void checkLastTagWas(final int value) - throws InvalidProtocolBufferException { - if (lastTag != value) { - throw InvalidProtocolBufferException.invalidEndTag(); - } - } + public abstract void checkLastTagWas(final int value) throws InvalidProtocolBufferException; - public int getLastTag() { - return lastTag; - } + public abstract int getLastTag(); /** * Reads and discards a single field, given its tag value. * - * @return {@code false} if the tag is an endgroup tag, in which case - * nothing is skipped. Otherwise, returns {@code true}. + * @return {@code false} if the tag is an endgroup tag, in which case nothing is skipped. + * Otherwise, returns {@code true}. */ - public boolean skipField(final int tag) throws IOException { - switch (WireFormat.getTagWireType(tag)) { - case WireFormat.WIRETYPE_VARINT: - skipRawVarint(); - return true; - case WireFormat.WIRETYPE_FIXED64: - skipRawBytes(8); - return true; - case WireFormat.WIRETYPE_LENGTH_DELIMITED: - skipRawBytes(readRawVarint32()); - return true; - case WireFormat.WIRETYPE_START_GROUP: - skipMessage(); - checkLastTagWas( - WireFormat.makeTag(WireFormat.getTagFieldNumber(tag), - WireFormat.WIRETYPE_END_GROUP)); - return true; - case WireFormat.WIRETYPE_END_GROUP: - return false; - case WireFormat.WIRETYPE_FIXED32: - skipRawBytes(4); - return true; - default: - throw InvalidProtocolBufferException.invalidWireType(); - } - } + public abstract boolean skipField(final int tag) throws IOException; /** - * Reads a single field and writes it to output in wire format, - * given its tag value. + * Reads a single field and writes it to output in wire format, given its tag value. * - * @return {@code false} if the tag is an endgroup tag, in which case - * nothing is skipped. Otherwise, returns {@code true}. + * @return {@code false} if the tag is an endgroup tag, in which case nothing is skipped. + * Otherwise, returns {@code true}. + * @deprecated use {@code UnknownFieldSet} or {@code UnknownFieldSetLite} to skip to an output + * stream. */ - public boolean skipField(final int tag, final CodedOutputStream output) - throws IOException { - switch (WireFormat.getTagWireType(tag)) { - case WireFormat.WIRETYPE_VARINT: { - long value = readInt64(); - output.writeRawVarint32(tag); - output.writeUInt64NoTag(value); - return true; - } - case WireFormat.WIRETYPE_FIXED64: { - long value = readRawLittleEndian64(); - output.writeRawVarint32(tag); - output.writeFixed64NoTag(value); - return true; - } - case WireFormat.WIRETYPE_LENGTH_DELIMITED: { - ByteString value = readBytes(); - output.writeRawVarint32(tag); - output.writeBytesNoTag(value); - return true; - } - case WireFormat.WIRETYPE_START_GROUP: { - output.writeRawVarint32(tag); - skipMessage(output); - int endtag = WireFormat.makeTag(WireFormat.getTagFieldNumber(tag), - WireFormat.WIRETYPE_END_GROUP); - checkLastTagWas(endtag); - output.writeRawVarint32(endtag); - return true; - } - case WireFormat.WIRETYPE_END_GROUP: { - return false; - } - case WireFormat.WIRETYPE_FIXED32: { - int value = readRawLittleEndian32(); - output.writeRawVarint32(tag); - output.writeFixed32NoTag(value); - return true; - } - default: - throw InvalidProtocolBufferException.invalidWireType(); - } - } - - /** - * Reads and discards an entire message. This will read either until EOF - * or until an endgroup tag, whichever comes first. - */ - public void skipMessage() throws IOException { - while (true) { - final int tag = readTag(); - if (tag == 0 || !skipField(tag)) { - return; - } - } - } + @Deprecated + public abstract boolean skipField(final int tag, final CodedOutputStream output) + throws IOException; /** - * Reads an entire message and writes it to output in wire format. - * This will read either until EOF or until an endgroup tag, + * Reads and discards an entire message. This will read either until EOF or until an endgroup tag, * whichever comes first. */ - public void skipMessage(CodedOutputStream output) throws IOException { - while (true) { - final int tag = readTag(); - if (tag == 0 || !skipField(tag, output)) { - return; - } - } - } + public abstract void skipMessage() throws IOException; /** - * Collects the bytes skipped and returns the data in a ByteBuffer. + * Reads an entire message and writes it to output in wire format. This will read either until EOF + * or until an endgroup tag, whichever comes first. */ - private class SkippedDataSink implements RefillCallback { - private int lastPos = bufferPos; - private ByteArrayOutputStream byteArrayStream; - - @Override - public void onRefill() { - if (byteArrayStream == null) { - byteArrayStream = new ByteArrayOutputStream(); - } - byteArrayStream.write(buffer, lastPos, bufferPos - lastPos); - lastPos = 0; - } - - /** - * Gets skipped data in a ByteBuffer. This method should only be - * called once. - */ - ByteBuffer getSkippedData() { - if (byteArrayStream == null) { - return ByteBuffer.wrap(buffer, lastPos, bufferPos - lastPos); - } else { - byteArrayStream.write(buffer, lastPos, bufferPos); - return ByteBuffer.wrap(byteArrayStream.toByteArray()); - } - } - } + public abstract void skipMessage(CodedOutputStream output) throws IOException; // ----------------------------------------------------------------- /** Read a {@code double} field value from the stream. */ - public double readDouble() throws IOException { - return Double.longBitsToDouble(readRawLittleEndian64()); - } + public abstract double readDouble() throws IOException; /** Read a {@code float} field value from the stream. */ - public float readFloat() throws IOException { - return Float.intBitsToFloat(readRawLittleEndian32()); - } + public abstract float readFloat() throws IOException; /** Read a {@code uint64} field value from the stream. */ - public long readUInt64() throws IOException { - return readRawVarint64(); - } + public abstract long readUInt64() throws IOException; /** Read an {@code int64} field value from the stream. */ - public long readInt64() throws IOException { - return readRawVarint64(); - } + public abstract long readInt64() throws IOException; /** Read an {@code int32} field value from the stream. */ - public int readInt32() throws IOException { - return readRawVarint32(); - } + public abstract int readInt32() throws IOException; /** Read a {@code fixed64} field value from the stream. */ - public long readFixed64() throws IOException { - return readRawLittleEndian64(); - } + public abstract long readFixed64() throws IOException; /** Read a {@code fixed32} field value from the stream. */ - public int readFixed32() throws IOException { - return readRawLittleEndian32(); - } + public abstract int readFixed32() throws IOException; /** Read a {@code bool} field value from the stream. */ - public boolean readBool() throws IOException { - return readRawVarint64() != 0; - } + public abstract boolean readBool() throws IOException; /** - * Read a {@code string} field value from the stream. - * If the stream contains malformed UTF-8, + * Read a {@code string} field value from the stream. If the stream contains malformed UTF-8, * replace the offending bytes with the standard UTF-8 replacement character. */ - public String readString() throws IOException { - final int size = readRawVarint32(); - if (size <= (bufferSize - bufferPos) && size > 0) { - // Fast path: We already have the bytes in a contiguous buffer, so - // just copy directly from it. - final String result = new String(buffer, bufferPos, size, Internal.UTF_8); - bufferPos += size; - return result; - } else if (size == 0) { - return ""; - } else if (size <= bufferSize) { - refillBuffer(size); - String result = new String(buffer, bufferPos, size, Internal.UTF_8); - bufferPos += size; - return result; - } else { - // Slow path: Build a byte array first then copy it. - return new String(readRawBytesSlowPath(size), Internal.UTF_8); - } - } + public abstract String readString() throws IOException; /** - * Read a {@code string} field value from the stream. - * If the stream contains malformed UTF-8, + * Read a {@code string} field value from the stream. If the stream contains malformed UTF-8, * throw exception {@link InvalidProtocolBufferException}. */ - public String readStringRequireUtf8() throws IOException { - final int size = readRawVarint32(); - final byte[] bytes; - final int oldPos = bufferPos; - final int pos; - if (size <= (bufferSize - oldPos) && size > 0) { - // Fast path: We already have the bytes in a contiguous buffer, so - // just copy directly from it. - bytes = buffer; - bufferPos = oldPos + size; - pos = oldPos; - } else if (size == 0) { - return ""; - } else if (size <= bufferSize) { - refillBuffer(size); - bytes = buffer; - pos = 0; - bufferPos = pos + size; - } else { - // Slow path: Build a byte array first then copy it. - bytes = readRawBytesSlowPath(size); - pos = 0; - } - // TODO(martinrb): We could save a pass by validating while decoding. - if (!Utf8.isValidUtf8(bytes, pos, pos + size)) { - throw InvalidProtocolBufferException.invalidUtf8(); - } - return new String(bytes, pos, size, Internal.UTF_8); - } + public abstract String readStringRequireUtf8() throws IOException; /** Read a {@code group} field value from the stream. */ - public void readGroup(final int fieldNumber, - final MessageLite.Builder builder, - final ExtensionRegistryLite extensionRegistry) - throws IOException { - if (recursionDepth >= recursionLimit) { - throw InvalidProtocolBufferException.recursionLimitExceeded(); - } - ++recursionDepth; - builder.mergeFrom(this, extensionRegistry); - checkLastTagWas( - WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); - --recursionDepth; - } + public abstract void readGroup( + final int fieldNumber, + final MessageLite.Builder builder, + final ExtensionRegistryLite extensionRegistry) + throws IOException; /** Read a {@code group} field value from the stream. */ - public <T extends MessageLite> T readGroup( - final int fieldNumber, - final Parser<T> parser, - final ExtensionRegistryLite extensionRegistry) - throws IOException { - if (recursionDepth >= recursionLimit) { - throw InvalidProtocolBufferException.recursionLimitExceeded(); - } - ++recursionDepth; - T result = parser.parsePartialFrom(this, extensionRegistry); - checkLastTagWas( - WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); - --recursionDepth; - return result; - } + public abstract <T extends MessageLite> T readGroup( + final int fieldNumber, final Parser<T> parser, final ExtensionRegistryLite extensionRegistry) + throws IOException; /** - * Reads a {@code group} field value from the stream and merges it into the - * given {@link UnknownFieldSet}. + * Reads a {@code group} field value from the stream and merges it into the given {@link + * UnknownFieldSet}. * - * @deprecated UnknownFieldSet.Builder now implements MessageLite.Builder, so - * you can just call {@link #readGroup}. + * @deprecated UnknownFieldSet.Builder now implements MessageLite.Builder, so you can just call + * {@link #readGroup}. */ @Deprecated - public void readUnknownGroup(final int fieldNumber, - final MessageLite.Builder builder) - throws IOException { - // We know that UnknownFieldSet will ignore any ExtensionRegistry so it - // is safe to pass null here. (We can't call - // ExtensionRegistry.getEmptyRegistry() because that would make this - // class depend on ExtensionRegistry, which is not part of the lite - // library.) - readGroup(fieldNumber, builder, null); - } + public abstract void readUnknownGroup(final int fieldNumber, final MessageLite.Builder builder) + throws IOException; /** Read an embedded message field value from the stream. */ - public void readMessage(final MessageLite.Builder builder, - final ExtensionRegistryLite extensionRegistry) - throws IOException { - final int length = readRawVarint32(); - if (recursionDepth >= recursionLimit) { - throw InvalidProtocolBufferException.recursionLimitExceeded(); - } - final int oldLimit = pushLimit(length); - ++recursionDepth; - builder.mergeFrom(this, extensionRegistry); - checkLastTagWas(0); - --recursionDepth; - popLimit(oldLimit); - } + public abstract void readMessage( + final MessageLite.Builder builder, final ExtensionRegistryLite extensionRegistry) + throws IOException; /** Read an embedded message field value from the stream. */ - public <T extends MessageLite> T readMessage( - final Parser<T> parser, - final ExtensionRegistryLite extensionRegistry) - throws IOException { - int length = readRawVarint32(); - if (recursionDepth >= recursionLimit) { - throw InvalidProtocolBufferException.recursionLimitExceeded(); - } - final int oldLimit = pushLimit(length); - ++recursionDepth; - T result = parser.parsePartialFrom(this, extensionRegistry); - checkLastTagWas(0); - --recursionDepth; - popLimit(oldLimit); - return result; - } + public abstract <T extends MessageLite> T readMessage( + final Parser<T> parser, final ExtensionRegistryLite extensionRegistry) throws IOException; /** Read a {@code bytes} field value from the stream. */ - public ByteString readBytes() throws IOException { - final int size = readRawVarint32(); - if (size <= (bufferSize - bufferPos) && size > 0) { - // Fast path: We already have the bytes in a contiguous buffer, so - // just copy directly from it. - final ByteString result = bufferIsImmutable && enableAliasing - ? ByteString.wrap(buffer, bufferPos, size) - : ByteString.copyFrom(buffer, bufferPos, size); - bufferPos += size; - return result; - } else if (size == 0) { - return ByteString.EMPTY; - } else { - // Slow path: Build a byte array first then copy it. - return ByteString.wrap(readRawBytesSlowPath(size)); - } - } + public abstract ByteString readBytes() throws IOException; /** Read a {@code bytes} field value from the stream. */ - public byte[] readByteArray() throws IOException { - final int size = readRawVarint32(); - if (size <= (bufferSize - bufferPos) && size > 0) { - // Fast path: We already have the bytes in a contiguous buffer, so - // just copy directly from it. - final byte[] result = - Arrays.copyOfRange(buffer, bufferPos, bufferPos + size); - bufferPos += size; - return result; - } else { - // Slow path: Build a byte array first then copy it. - return readRawBytesSlowPath(size); - } - } + public abstract byte[] readByteArray() throws IOException; /** Read a {@code bytes} field value from the stream. */ - public ByteBuffer readByteBuffer() throws IOException { - final int size = readRawVarint32(); - if (size <= (bufferSize - bufferPos) && size > 0) { - // Fast path: We already have the bytes in a contiguous buffer. - // When aliasing is enabled, we can return a ByteBuffer pointing directly - // into the underlying byte array without copy if the CodedInputStream is - // constructed from a byte array. If aliasing is disabled or the input is - // from an InputStream or ByteString, we have to make a copy of the bytes. - ByteBuffer result = input == null && !bufferIsImmutable && enableAliasing - ? ByteBuffer.wrap(buffer, bufferPos, size).slice() - : ByteBuffer.wrap(Arrays.copyOfRange( - buffer, bufferPos, bufferPos + size)); - bufferPos += size; - return result; - } else if (size == 0) { - return Internal.EMPTY_BYTE_BUFFER; - } else { - // Slow path: Build a byte array first then copy it. - return ByteBuffer.wrap(readRawBytesSlowPath(size)); - } - } + public abstract ByteBuffer readByteBuffer() throws IOException; /** Read a {@code uint32} field value from the stream. */ - public int readUInt32() throws IOException { - return readRawVarint32(); - } + public abstract int readUInt32() throws IOException; /** - * Read an enum field value from the stream. Caller is responsible - * for converting the numeric value to an actual enum. + * Read an enum field value from the stream. Caller is responsible for converting the numeric + * value to an actual enum. */ - public int readEnum() throws IOException { - return readRawVarint32(); - } + public abstract int readEnum() throws IOException; /** Read an {@code sfixed32} field value from the stream. */ - public int readSFixed32() throws IOException { - return readRawLittleEndian32(); - } + public abstract int readSFixed32() throws IOException; /** Read an {@code sfixed64} field value from the stream. */ - public long readSFixed64() throws IOException { - return readRawLittleEndian64(); - } + public abstract long readSFixed64() throws IOException; /** Read an {@code sint32} field value from the stream. */ - public int readSInt32() throws IOException { - return decodeZigZag32(readRawVarint32()); - } + public abstract int readSInt32() throws IOException; /** Read an {@code sint64} field value from the stream. */ - public long readSInt64() throws IOException { - return decodeZigZag64(readRawVarint64()); - } + public abstract long readSInt64() throws IOException; // ================================================================= + /** Read a raw Varint from the stream. If larger than 32 bits, discard the upper bits. */ + public abstract int readRawVarint32() throws IOException; + + /** Read a raw Varint from the stream. */ + public abstract long readRawVarint64() throws IOException; + + /** Variant of readRawVarint64 for when uncomfortably close to the limit. */ + /* Visible for testing */ + abstract long readRawVarint64SlowPath() throws IOException; + + /** Read a 32-bit little-endian integer from the stream. */ + public abstract int readRawLittleEndian32() throws IOException; + + /** Read a 64-bit little-endian integer from the stream. */ + public abstract long readRawLittleEndian64() throws IOException; + + // ----------------------------------------------------------------- + /** - * Read a raw Varint from the stream. If larger than 32 bits, discard the - * upper bits. + * Enables {@link ByteString} aliasing of the underlying buffer, trading off on buffer pinning for + * data copies. Only valid for buffer-backed streams. */ - public int readRawVarint32() throws IOException { - // See implementation notes for readRawVarint64 - fastpath: { - int pos = bufferPos; - - if (bufferSize == pos) { - break fastpath; - } + public abstract void enableAliasing(boolean enabled); - final byte[] buffer = this.buffer; - int x; - if ((x = buffer[pos++]) >= 0) { - bufferPos = pos; - return x; - } else if (bufferSize - pos < 9) { - break fastpath; - } else if ((x ^= (buffer[pos++] << 7)) < 0) { - x ^= (~0 << 7); - } else if ((x ^= (buffer[pos++] << 14)) >= 0) { - x ^= (~0 << 7) ^ (~0 << 14); - } else if ((x ^= (buffer[pos++] << 21)) < 0) { - x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21); - } else { - int y = buffer[pos++]; - x ^= y << 28; - x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21) ^ (~0 << 28); - if (y < 0 && - buffer[pos++] < 0 && - buffer[pos++] < 0 && - buffer[pos++] < 0 && - buffer[pos++] < 0 && - buffer[pos++] < 0) { - break fastpath; // Will throw malformedVarint() - } - } - bufferPos = pos; - return x; + /** + * Set the maximum message recursion depth. In order to prevent malicious messages from causing + * stack overflows, {@code CodedInputStream} limits how deeply messages may be nested. The default + * limit is 64. + * + * @return the old limit. + */ + public final int setRecursionLimit(final int limit) { + if (limit < 0) { + throw new IllegalArgumentException("Recursion limit cannot be negative: " + limit); } - return (int) readRawVarint64SlowPath(); + final int oldLimit = recursionLimit; + recursionLimit = limit; + return oldLimit; } - private void skipRawVarint() throws IOException { - if (bufferSize - bufferPos >= 10) { - final byte[] buffer = this.buffer; - int pos = bufferPos; - for (int i = 0; i < 10; i++) { - if (buffer[pos++] >= 0) { - bufferPos = pos; - return; - } - } + /** + * Only valid for {@link InputStream}-backed streams. + * + * <p>Set the maximum message size. In order to prevent malicious messages from exhausting memory + * or causing integer overflows, {@code CodedInputStream} limits how large a message may be. The + * default limit is 64MB. You should set this limit as small as you can without harming your app's + * functionality. Note that size limits only apply when reading from an {@code InputStream}, not + * when constructed around a raw byte array (nor with {@link ByteString#newCodedInput}). + * + * <p>If you want to read several messages from a single CodedInputStream, you could call {@link + * #resetSizeCounter()} after each one to avoid hitting the size limit. + * + * @return the old limit. + */ + public final int setSizeLimit(final int limit) { + if (limit < 0) { + throw new IllegalArgumentException("Size limit cannot be negative: " + limit); } - skipRawVarintSlowPath(); + final int oldLimit = sizeLimit; + sizeLimit = limit; + return oldLimit; } - private void skipRawVarintSlowPath() throws IOException { - for (int i = 0; i < 10; i++) { - if (readRawByte() >= 0) { - return; - } - } - throw InvalidProtocolBufferException.malformedVarint(); + /** + * Resets the current size counter to zero (see {@link #setSizeLimit(int)}). Only valid for {@link + * InputStream}-backed streams. + */ + public abstract void resetSizeCounter(); + + /** + * Sets {@code currentLimit} to (current position) + {@code byteLimit}. This is called when + * descending into a length-delimited embedded message. + * + * <p>Note that {@code pushLimit()} does NOT affect how many bytes the {@code CodedInputStream} + * reads from an underlying {@code InputStream} when refreshing its buffer. If you need to prevent + * reading past a certain point in the underlying {@code InputStream} (e.g. because you expect it + * to contain more data after the end of the message which you need to handle differently) then + * you must place a wrapper around your {@code InputStream} which limits the amount of data that + * can be read from it. + * + * @return the old limit. + */ + public abstract int pushLimit(int byteLimit) throws InvalidProtocolBufferException; + + /** + * Discards the current limit, returning to the previous limit. + * + * @param oldLimit The old limit, as returned by {@code pushLimit}. + */ + public abstract void popLimit(final int oldLimit); + + /** + * Returns the number of bytes to be read before the current limit. If no limit is set, returns + * -1. + */ + public abstract int getBytesUntilLimit(); + + /** + * Returns true if the stream has reached the end of the input. This is the case if either the end + * of the underlying input source has been reached or if the stream has reached a limit created + * using {@link #pushLimit(int)}. + */ + public abstract boolean isAtEnd() throws IOException; + + /** + * The total bytes read up to the current position. Calling {@link #resetSizeCounter()} resets + * this value to zero. + */ + public abstract int getTotalBytesRead(); + + /** + * Read one byte from the input. + * + * @throws InvalidProtocolBufferException The end of the stream or the current limit was reached. + */ + public abstract byte readRawByte() throws IOException; + + /** + * Read a fixed size of bytes from the input. + * + * @throws InvalidProtocolBufferException The end of the stream or the current limit was reached. + */ + public abstract byte[] readRawBytes(final int size) throws IOException; + + /** + * Reads and discards {@code size} bytes. + * + * @throws InvalidProtocolBufferException The end of the stream or the current limit was reached. + */ + public abstract void skipRawBytes(final int size) throws IOException; + + /** + * Decode a ZigZag-encoded 32-bit value. ZigZag encodes signed integers into values that can be + * efficiently encoded with varint. (Otherwise, negative values must be sign-extended to 64 bits + * to be varint encoded, thus always taking 10 bytes on the wire.) + * + * @param n An unsigned 32-bit integer, stored in a signed int because Java has no explicit + * unsigned support. + * @return A signed 32-bit integer. + */ + public static int decodeZigZag32(final int n) { + return (n >>> 1) ^ -(n & 1); } /** - * Reads a varint from the input one byte at a time, so that it does not - * read any bytes after the end of the varint. If you simply wrapped the - * stream in a CodedInputStream and used {@link #readRawVarint32(InputStream)} - * then you would probably end up reading past the end of the varint since - * CodedInputStream buffers its input. + * Decode a ZigZag-encoded 64-bit value. ZigZag encodes signed integers into values that can be + * efficiently encoded with varint. (Otherwise, negative values must be sign-extended to 64 bits + * to be varint encoded, thus always taking 10 bytes on the wire.) + * + * @param n An unsigned 64-bit integer, stored in a signed int because Java has no explicit + * unsigned support. + * @return A signed 64-bit integer. */ - static int readRawVarint32(final InputStream input) throws IOException { - final int firstByte = input.read(); - if (firstByte == -1) { - throw InvalidProtocolBufferException.truncatedMessage(); - } - return readRawVarint32(firstByte, input); + public static long decodeZigZag64(final long n) { + return (n >>> 1) ^ -(n & 1); } /** - * Like {@link #readRawVarint32(InputStream)}, but expects that the caller - * has already read one byte. This allows the caller to determine if EOF - * has been reached before attempting to read. + * Like {@link #readRawVarint32(InputStream)}, but expects that the caller has already read one + * byte. This allows the caller to determine if EOF has been reached before attempting to read. */ - public static int readRawVarint32( - final int firstByte, final InputStream input) throws IOException { + public static int readRawVarint32(final int firstByte, final InputStream input) + throws IOException { if ((firstByte & 0x80) == 0) { return firstByte; } @@ -713,590 +500,2396 @@ public final class CodedInputStream { throw InvalidProtocolBufferException.malformedVarint(); } - /** Read a raw Varint from the stream. */ - public long readRawVarint64() throws IOException { - // Implementation notes: - // - // Optimized for one-byte values, expected to be common. - // The particular code below was selected from various candidates - // empirically, by winning VarintBenchmark. - // - // Sign extension of (signed) Java bytes is usually a nuisance, but - // we exploit it here to more easily obtain the sign of bytes read. - // Instead of cleaning up the sign extension bits by masking eagerly, - // we delay until we find the final (positive) byte, when we clear all - // accumulated bits with one xor. We depend on javac to constant fold. - fastpath: { - int pos = bufferPos; - - if (bufferSize == pos) { - break fastpath; + /** + * Reads a varint from the input one byte at a time, so that it does not read any bytes after the + * end of the varint. If you simply wrapped the stream in a CodedInputStream and used {@link + * #readRawVarint32(InputStream)} then you would probably end up reading past the end of the + * varint since CodedInputStream buffers its input. + */ + static int readRawVarint32(final InputStream input) throws IOException { + final int firstByte = input.read(); + if (firstByte == -1) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + return readRawVarint32(firstByte, input); + } + + /** A {@link CodedInputStream} implementation that uses a backing array as the input. */ + private static final class ArrayDecoder extends CodedInputStream { + private final byte[] buffer; + private final boolean immutable; + private int limit; + private int bufferSizeAfterLimit; + private int pos; + private int startPos; + private int lastTag; + private boolean enableAliasing; + + /** The absolute position of the end of the current message. */ + private int currentLimit = Integer.MAX_VALUE; + + private ArrayDecoder(final byte[] buffer, final int offset, final int len, boolean immutable) { + this.buffer = buffer; + limit = offset + len; + pos = offset; + startPos = pos; + this.immutable = immutable; + } + + @Override + public int readTag() throws IOException { + if (isAtEnd()) { + lastTag = 0; + return 0; } - final byte[] buffer = this.buffer; - long x; - int y; - if ((y = buffer[pos++]) >= 0) { - bufferPos = pos; - return y; - } else if (bufferSize - pos < 9) { - break fastpath; - } else if ((y ^= (buffer[pos++] << 7)) < 0) { - x = y ^ (~0 << 7); - } else if ((y ^= (buffer[pos++] << 14)) >= 0) { - x = y ^ ((~0 << 7) ^ (~0 << 14)); - } else if ((y ^= (buffer[pos++] << 21)) < 0) { - x = y ^ ((~0 << 7) ^ (~0 << 14) ^ (~0 << 21)); - } else if ((x = ((long) y) ^ ((long) buffer[pos++] << 28)) >= 0L) { - x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28); - } else if ((x ^= ((long) buffer[pos++] << 35)) < 0L) { - x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35); - } else if ((x ^= ((long) buffer[pos++] << 42)) >= 0L) { - x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35) ^ (~0L << 42); - } else if ((x ^= ((long) buffer[pos++] << 49)) < 0L) { - x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35) ^ (~0L << 42) - ^ (~0L << 49); - } else { - x ^= ((long) buffer[pos++] << 56); - x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35) ^ (~0L << 42) - ^ (~0L << 49) ^ (~0L << 56); - if (x < 0L) { - if (buffer[pos++] < 0L) { - break fastpath; // Will throw malformedVarint() + lastTag = readRawVarint32(); + if (WireFormat.getTagFieldNumber(lastTag) == 0) { + // If we actually read zero (or any tag number corresponding to field + // number zero), that's not a valid tag. + throw InvalidProtocolBufferException.invalidTag(); + } + return lastTag; + } + + @Override + public void checkLastTagWas(final int value) throws InvalidProtocolBufferException { + if (lastTag != value) { + throw InvalidProtocolBufferException.invalidEndTag(); + } + } + + @Override + public int getLastTag() { + return lastTag; + } + + @Override + public boolean skipField(final int tag) throws IOException { + switch (WireFormat.getTagWireType(tag)) { + case WireFormat.WIRETYPE_VARINT: + skipRawVarint(); + return true; + case WireFormat.WIRETYPE_FIXED64: + skipRawBytes(FIXED_64_SIZE); + return true; + case WireFormat.WIRETYPE_LENGTH_DELIMITED: + skipRawBytes(readRawVarint32()); + return true; + case WireFormat.WIRETYPE_START_GROUP: + skipMessage(); + checkLastTagWas( + WireFormat.makeTag(WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP)); + return true; + case WireFormat.WIRETYPE_END_GROUP: + return false; + case WireFormat.WIRETYPE_FIXED32: + skipRawBytes(FIXED_32_SIZE); + return true; + default: + throw InvalidProtocolBufferException.invalidWireType(); + } + } + + @Override + public boolean skipField(final int tag, final CodedOutputStream output) throws IOException { + switch (WireFormat.getTagWireType(tag)) { + case WireFormat.WIRETYPE_VARINT: + { + long value = readInt64(); + output.writeRawVarint32(tag); + output.writeUInt64NoTag(value); + return true; + } + case WireFormat.WIRETYPE_FIXED64: + { + long value = readRawLittleEndian64(); + output.writeRawVarint32(tag); + output.writeFixed64NoTag(value); + return true; + } + case WireFormat.WIRETYPE_LENGTH_DELIMITED: + { + ByteString value = readBytes(); + output.writeRawVarint32(tag); + output.writeBytesNoTag(value); + return true; + } + case WireFormat.WIRETYPE_START_GROUP: + { + output.writeRawVarint32(tag); + skipMessage(output); + int endtag = + WireFormat.makeTag( + WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP); + checkLastTagWas(endtag); + output.writeRawVarint32(endtag); + return true; } + case WireFormat.WIRETYPE_END_GROUP: + { + return false; + } + case WireFormat.WIRETYPE_FIXED32: + { + int value = readRawLittleEndian32(); + output.writeRawVarint32(tag); + output.writeFixed32NoTag(value); + return true; + } + default: + throw InvalidProtocolBufferException.invalidWireType(); + } + } + + @Override + public void skipMessage() throws IOException { + while (true) { + final int tag = readTag(); + if (tag == 0 || !skipField(tag)) { + return; } } - bufferPos = pos; - return x; } - return readRawVarint64SlowPath(); - } - /** Variant of readRawVarint64 for when uncomfortably close to the limit. */ - /* Visible for testing */ - long readRawVarint64SlowPath() throws IOException { - long result = 0; - for (int shift = 0; shift < 64; shift += 7) { - final byte b = readRawByte(); - result |= (long) (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - return result; + @Override + public void skipMessage(CodedOutputStream output) throws IOException { + while (true) { + final int tag = readTag(); + if (tag == 0 || !skipField(tag, output)) { + return; + } } } - throw InvalidProtocolBufferException.malformedVarint(); - } - /** Read a 32-bit little-endian integer from the stream. */ - public int readRawLittleEndian32() throws IOException { - int pos = bufferPos; - // hand-inlined ensureAvailable(4); - if (bufferSize - pos < 4) { - refillBuffer(4); - pos = bufferPos; + // ----------------------------------------------------------------- + + @Override + public double readDouble() throws IOException { + return Double.longBitsToDouble(readRawLittleEndian64()); } - final byte[] buffer = this.buffer; - bufferPos = pos + 4; - return (((buffer[pos] & 0xff)) | - ((buffer[pos + 1] & 0xff) << 8) | - ((buffer[pos + 2] & 0xff) << 16) | - ((buffer[pos + 3] & 0xff) << 24)); - } + @Override + public float readFloat() throws IOException { + return Float.intBitsToFloat(readRawLittleEndian32()); + } - /** Read a 64-bit little-endian integer from the stream. */ - public long readRawLittleEndian64() throws IOException { - int pos = bufferPos; - - // hand-inlined ensureAvailable(8); - if (bufferSize - pos < 8) { - refillBuffer(8); - pos = bufferPos; - } - - final byte[] buffer = this.buffer; - bufferPos = pos + 8; - return ((((long) buffer[pos] & 0xffL)) | - (((long) buffer[pos + 1] & 0xffL) << 8) | - (((long) buffer[pos + 2] & 0xffL) << 16) | - (((long) buffer[pos + 3] & 0xffL) << 24) | - (((long) buffer[pos + 4] & 0xffL) << 32) | - (((long) buffer[pos + 5] & 0xffL) << 40) | - (((long) buffer[pos + 6] & 0xffL) << 48) | - (((long) buffer[pos + 7] & 0xffL) << 56)); - } + @Override + public long readUInt64() throws IOException { + return readRawVarint64(); + } - /** - * Decode a ZigZag-encoded 32-bit value. ZigZag encodes signed integers - * into values that can be efficiently encoded with varint. (Otherwise, - * negative values must be sign-extended to 64 bits to be varint encoded, - * thus always taking 10 bytes on the wire.) - * - * @param n An unsigned 32-bit integer, stored in a signed int because - * Java has no explicit unsigned support. - * @return A signed 32-bit integer. - */ - public static int decodeZigZag32(final int n) { - return (n >>> 1) ^ -(n & 1); - } + @Override + public long readInt64() throws IOException { + return readRawVarint64(); + } - /** - * Decode a ZigZag-encoded 64-bit value. ZigZag encodes signed integers - * into values that can be efficiently encoded with varint. (Otherwise, - * negative values must be sign-extended to 64 bits to be varint encoded, - * thus always taking 10 bytes on the wire.) - * - * @param n An unsigned 64-bit integer, stored in a signed int because - * Java has no explicit unsigned support. - * @return A signed 64-bit integer. - */ - public static long decodeZigZag64(final long n) { - return (n >>> 1) ^ -(n & 1); - } + @Override + public int readInt32() throws IOException { + return readRawVarint32(); + } - // ----------------------------------------------------------------- + @Override + public long readFixed64() throws IOException { + return readRawLittleEndian64(); + } - private final byte[] buffer; - private final boolean bufferIsImmutable; - private int bufferSize; - private int bufferSizeAfterLimit; - private int bufferPos; - private final InputStream input; - private int lastTag; - private boolean enableAliasing = false; + @Override + public int readFixed32() throws IOException { + return readRawLittleEndian32(); + } - /** - * The total number of bytes read before the current buffer. The total - * bytes read up to the current position can be computed as - * {@code totalBytesRetired + bufferPos}. This value may be negative if - * reading started in the middle of the current buffer (e.g. if the - * constructor that takes a byte array and an offset was used). - */ - private int totalBytesRetired; + @Override + public boolean readBool() throws IOException { + return readRawVarint64() != 0; + } - /** The absolute position of the end of the current message. */ - private int currentLimit = Integer.MAX_VALUE; + @Override + public String readString() throws IOException { + final int size = readRawVarint32(); + if (size > 0 && size <= (limit - pos)) { + // Fast path: We already have the bytes in a contiguous buffer, so + // just copy directly from it. + final String result = new String(buffer, pos, size, UTF_8); + pos += size; + return result; + } - /** See setRecursionLimit() */ - private int recursionDepth; - private int recursionLimit = DEFAULT_RECURSION_LIMIT; + if (size == 0) { + return ""; + } + if (size < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + throw InvalidProtocolBufferException.truncatedMessage(); + } - /** See setSizeLimit() */ - private int sizeLimit = DEFAULT_SIZE_LIMIT; + @Override + public String readStringRequireUtf8() throws IOException { + final int size = readRawVarint32(); + if (size > 0 && size <= (limit - pos)) { + // TODO(martinrb): We could save a pass by validating while decoding. + if (!Utf8.isValidUtf8(buffer, pos, pos + size)) { + throw InvalidProtocolBufferException.invalidUtf8(); + } + final int tempPos = pos; + pos += size; + return new String(buffer, tempPos, size, UTF_8); + } - private static final int DEFAULT_RECURSION_LIMIT = 100; - private static final int DEFAULT_SIZE_LIMIT = 64 << 20; // 64MB - private static final int BUFFER_SIZE = 4096; - - private CodedInputStream( - final byte[] buffer, final int off, final int len, boolean bufferIsImmutable) { - this.buffer = buffer; - bufferSize = off + len; - bufferPos = off; - totalBytesRetired = -off; - input = null; - this.bufferIsImmutable = bufferIsImmutable; - } + if (size == 0) { + return ""; + } + if (size <= 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + throw InvalidProtocolBufferException.truncatedMessage(); + } - private CodedInputStream(final InputStream input, int bufferSize) { - buffer = new byte[bufferSize]; - bufferSize = 0; - bufferPos = 0; - totalBytesRetired = 0; - this.input = input; - bufferIsImmutable = false; - } + @Override + public void readGroup( + final int fieldNumber, + final MessageLite.Builder builder, + final ExtensionRegistryLite extensionRegistry) + throws IOException { + if (recursionDepth >= recursionLimit) { + throw InvalidProtocolBufferException.recursionLimitExceeded(); + } + ++recursionDepth; + builder.mergeFrom(this, extensionRegistry); + checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); + --recursionDepth; + } - public void enableAliasing(boolean enabled) { - this.enableAliasing = enabled; - } - /** - * Set the maximum message recursion depth. In order to prevent malicious - * messages from causing stack overflows, {@code CodedInputStream} limits - * how deeply messages may be nested. The default limit is 64. - * - * @return the old limit. - */ - public int setRecursionLimit(final int limit) { - if (limit < 0) { - throw new IllegalArgumentException( - "Recursion limit cannot be negative: " + limit); + @Override + public <T extends MessageLite> T readGroup( + final int fieldNumber, + final Parser<T> parser, + final ExtensionRegistryLite extensionRegistry) + throws IOException { + if (recursionDepth >= recursionLimit) { + throw InvalidProtocolBufferException.recursionLimitExceeded(); + } + ++recursionDepth; + T result = parser.parsePartialFrom(this, extensionRegistry); + checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); + --recursionDepth; + return result; } - final int oldLimit = recursionLimit; - recursionLimit = limit; - return oldLimit; - } - /** - * Set the maximum message size. In order to prevent malicious - * messages from exhausting memory or causing integer overflows, - * {@code CodedInputStream} limits how large a message may be. - * The default limit is 64MB. You should set this limit as small - * as you can without harming your app's functionality. Note that - * size limits only apply when reading from an {@code InputStream}, not - * when constructed around a raw byte array (nor with - * {@link ByteString#newCodedInput}). - * <p> - * If you want to read several messages from a single CodedInputStream, you - * could call {@link #resetSizeCounter()} after each one to avoid hitting the - * size limit. - * - * @return the old limit. - */ - public int setSizeLimit(final int limit) { - if (limit < 0) { - throw new IllegalArgumentException( - "Size limit cannot be negative: " + limit); + @Deprecated + @Override + public void readUnknownGroup(final int fieldNumber, final MessageLite.Builder builder) + throws IOException { + readGroup(fieldNumber, builder, ExtensionRegistryLite.getEmptyRegistry()); } - final int oldLimit = sizeLimit; - sizeLimit = limit; - return oldLimit; - } - /** - * Resets the current size counter to zero (see {@link #setSizeLimit(int)}). - */ - public void resetSizeCounter() { - totalBytesRetired = -bufferPos; - } + @Override + public void readMessage( + final MessageLite.Builder builder, final ExtensionRegistryLite extensionRegistry) + throws IOException { + final int length = readRawVarint32(); + if (recursionDepth >= recursionLimit) { + throw InvalidProtocolBufferException.recursionLimitExceeded(); + } + final int oldLimit = pushLimit(length); + ++recursionDepth; + builder.mergeFrom(this, extensionRegistry); + checkLastTagWas(0); + --recursionDepth; + popLimit(oldLimit); + } - /** - * Sets {@code currentLimit} to (current position) + {@code byteLimit}. This - * is called when descending into a length-delimited embedded message. - * - * <p>Note that {@code pushLimit()} does NOT affect how many bytes the - * {@code CodedInputStream} reads from an underlying {@code InputStream} when - * refreshing its buffer. If you need to prevent reading past a certain - * point in the underlying {@code InputStream} (e.g. because you expect it to - * contain more data after the end of the message which you need to handle - * differently) then you must place a wrapper around your {@code InputStream} - * which limits the amount of data that can be read from it. - * - * @return the old limit. - */ - public int pushLimit(int byteLimit) throws InvalidProtocolBufferException { - if (byteLimit < 0) { - throw InvalidProtocolBufferException.negativeSize(); + + @Override + public <T extends MessageLite> T readMessage( + final Parser<T> parser, final ExtensionRegistryLite extensionRegistry) throws IOException { + int length = readRawVarint32(); + if (recursionDepth >= recursionLimit) { + throw InvalidProtocolBufferException.recursionLimitExceeded(); + } + final int oldLimit = pushLimit(length); + ++recursionDepth; + T result = parser.parsePartialFrom(this, extensionRegistry); + checkLastTagWas(0); + --recursionDepth; + popLimit(oldLimit); + return result; + } + + @Override + public ByteString readBytes() throws IOException { + final int size = readRawVarint32(); + if (size > 0 && size <= (limit - pos)) { + // Fast path: We already have the bytes in a contiguous buffer, so + // just copy directly from it. + final ByteString result = + immutable && enableAliasing + ? ByteString.wrap(buffer, pos, size) + : ByteString.copyFrom(buffer, pos, size); + pos += size; + return result; + } + if (size == 0) { + return ByteString.EMPTY; + } + // Slow path: Build a byte array first then copy it. + return ByteString.wrap(readRawBytes(size)); + } + + @Override + public byte[] readByteArray() throws IOException { + final int size = readRawVarint32(); + return readRawBytes(size); } - byteLimit += totalBytesRetired + bufferPos; - final int oldLimit = currentLimit; - if (byteLimit > oldLimit) { + + @Override + public ByteBuffer readByteBuffer() throws IOException { + final int size = readRawVarint32(); + if (size > 0 && size <= (limit - pos)) { + // Fast path: We already have the bytes in a contiguous buffer. + // When aliasing is enabled, we can return a ByteBuffer pointing directly + // into the underlying byte array without copy if the CodedInputStream is + // constructed from a byte array. If aliasing is disabled or the input is + // from an InputStream or ByteString, we have to make a copy of the bytes. + ByteBuffer result = + !immutable && enableAliasing + ? ByteBuffer.wrap(buffer, pos, size).slice() + : ByteBuffer.wrap(Arrays.copyOfRange(buffer, pos, pos + size)); + pos += size; + // TODO(nathanmittler): Investigate making the ByteBuffer be made read-only + return result; + } + + if (size == 0) { + return EMPTY_BYTE_BUFFER; + } + if (size < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } throw InvalidProtocolBufferException.truncatedMessage(); } - currentLimit = byteLimit; - recomputeBufferSizeAfterLimit(); + @Override + public int readUInt32() throws IOException { + return readRawVarint32(); + } - return oldLimit; - } + @Override + public int readEnum() throws IOException { + return readRawVarint32(); + } - private void recomputeBufferSizeAfterLimit() { - bufferSize += bufferSizeAfterLimit; - final int bufferEnd = totalBytesRetired + bufferSize; - if (bufferEnd > currentLimit) { - // Limit is in current buffer. - bufferSizeAfterLimit = bufferEnd - currentLimit; - bufferSize -= bufferSizeAfterLimit; - } else { - bufferSizeAfterLimit = 0; + @Override + public int readSFixed32() throws IOException { + return readRawLittleEndian32(); } - } - /** - * Discards the current limit, returning to the previous limit. - * - * @param oldLimit The old limit, as returned by {@code pushLimit}. - */ - public void popLimit(final int oldLimit) { - currentLimit = oldLimit; - recomputeBufferSizeAfterLimit(); - } + @Override + public long readSFixed64() throws IOException { + return readRawLittleEndian64(); + } - /** - * Returns the number of bytes to be read before the current limit. - * If no limit is set, returns -1. - */ - public int getBytesUntilLimit() { - if (currentLimit == Integer.MAX_VALUE) { - return -1; + @Override + public int readSInt32() throws IOException { + return decodeZigZag32(readRawVarint32()); } - final int currentAbsolutePosition = totalBytesRetired + bufferPos; - return currentLimit - currentAbsolutePosition; - } + @Override + public long readSInt64() throws IOException { + return decodeZigZag64(readRawVarint64()); + } - /** - * Returns true if the stream has reached the end of the input. This is the - * case if either the end of the underlying input source has been reached or - * if the stream has reached a limit created using {@link #pushLimit(int)}. - */ - public boolean isAtEnd() throws IOException { - return bufferPos == bufferSize && !tryRefillBuffer(1); - } + // ================================================================= - /** - * The total bytes read up to the current position. Calling - * {@link #resetSizeCounter()} resets this value to zero. - */ - public int getTotalBytesRead() { - return totalBytesRetired + bufferPos; - } + @Override + public int readRawVarint32() throws IOException { + // See implementation notes for readRawVarint64 + fastpath: + { + int tempPos = pos; + + if (limit == tempPos) { + break fastpath; + } - private interface RefillCallback { - void onRefill(); - } + final byte[] buffer = this.buffer; + int x; + if ((x = buffer[tempPos++]) >= 0) { + pos = tempPos; + return x; + } else if (limit - tempPos < 9) { + break fastpath; + } else if ((x ^= (buffer[tempPos++] << 7)) < 0) { + x ^= (~0 << 7); + } else if ((x ^= (buffer[tempPos++] << 14)) >= 0) { + x ^= (~0 << 7) ^ (~0 << 14); + } else if ((x ^= (buffer[tempPos++] << 21)) < 0) { + x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21); + } else { + int y = buffer[tempPos++]; + x ^= y << 28; + x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21) ^ (~0 << 28); + if (y < 0 + && buffer[tempPos++] < 0 + && buffer[tempPos++] < 0 + && buffer[tempPos++] < 0 + && buffer[tempPos++] < 0 + && buffer[tempPos++] < 0) { + break fastpath; // Will throw malformedVarint() + } + } + pos = tempPos; + return x; + } + return (int) readRawVarint64SlowPath(); + } - private RefillCallback refillCallback = null; + private void skipRawVarint() throws IOException { + if (limit - pos >= MAX_VARINT_SIZE) { + skipRawVarintFastPath(); + } else { + skipRawVarintSlowPath(); + } + } - /** - * Reads more bytes from the input, making at least {@code n} bytes available - * in the buffer. Caller must ensure that the requested space is not yet - * available, and that the requested space is less than BUFFER_SIZE. - * - * @throws InvalidProtocolBufferException The end of the stream or the current - * limit was reached. - */ - private void refillBuffer(int n) throws IOException { - if (!tryRefillBuffer(n)) { + private void skipRawVarintFastPath() throws IOException { + for (int i = 0; i < MAX_VARINT_SIZE; i++) { + if (buffer[pos++] >= 0) { + return; + } + } + throw InvalidProtocolBufferException.malformedVarint(); + } + + private void skipRawVarintSlowPath() throws IOException { + for (int i = 0; i < MAX_VARINT_SIZE; i++) { + if (readRawByte() >= 0) { + return; + } + } + throw InvalidProtocolBufferException.malformedVarint(); + } + + @Override + public long readRawVarint64() throws IOException { + // Implementation notes: + // + // Optimized for one-byte values, expected to be common. + // The particular code below was selected from various candidates + // empirically, by winning VarintBenchmark. + // + // Sign extension of (signed) Java bytes is usually a nuisance, but + // we exploit it here to more easily obtain the sign of bytes read. + // Instead of cleaning up the sign extension bits by masking eagerly, + // we delay until we find the final (positive) byte, when we clear all + // accumulated bits with one xor. We depend on javac to constant fold. + fastpath: + { + int tempPos = pos; + + if (limit == tempPos) { + break fastpath; + } + + final byte[] buffer = this.buffer; + long x; + int y; + if ((y = buffer[tempPos++]) >= 0) { + pos = tempPos; + return y; + } else if (limit - tempPos < 9) { + break fastpath; + } else if ((y ^= (buffer[tempPos++] << 7)) < 0) { + x = y ^ (~0 << 7); + } else if ((y ^= (buffer[tempPos++] << 14)) >= 0) { + x = y ^ ((~0 << 7) ^ (~0 << 14)); + } else if ((y ^= (buffer[tempPos++] << 21)) < 0) { + x = y ^ ((~0 << 7) ^ (~0 << 14) ^ (~0 << 21)); + } else if ((x = y ^ ((long) buffer[tempPos++] << 28)) >= 0L) { + x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28); + } else if ((x ^= ((long) buffer[tempPos++] << 35)) < 0L) { + x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35); + } else if ((x ^= ((long) buffer[tempPos++] << 42)) >= 0L) { + x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35) ^ (~0L << 42); + } else if ((x ^= ((long) buffer[tempPos++] << 49)) < 0L) { + x ^= + (~0L << 7) + ^ (~0L << 14) + ^ (~0L << 21) + ^ (~0L << 28) + ^ (~0L << 35) + ^ (~0L << 42) + ^ (~0L << 49); + } else { + x ^= ((long) buffer[tempPos++] << 56); + x ^= + (~0L << 7) + ^ (~0L << 14) + ^ (~0L << 21) + ^ (~0L << 28) + ^ (~0L << 35) + ^ (~0L << 42) + ^ (~0L << 49) + ^ (~0L << 56); + if (x < 0L) { + if (buffer[tempPos++] < 0L) { + break fastpath; // Will throw malformedVarint() + } + } + } + pos = tempPos; + return x; + } + return readRawVarint64SlowPath(); + } + + @Override + long readRawVarint64SlowPath() throws IOException { + long result = 0; + for (int shift = 0; shift < 64; shift += 7) { + final byte b = readRawByte(); + result |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return result; + } + } + throw InvalidProtocolBufferException.malformedVarint(); + } + + @Override + public int readRawLittleEndian32() throws IOException { + int tempPos = pos; + + if (limit - tempPos < FIXED_32_SIZE) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + + final byte[] buffer = this.buffer; + pos = tempPos + FIXED_32_SIZE; + return (((buffer[tempPos] & 0xff)) + | ((buffer[tempPos + 1] & 0xff) << 8) + | ((buffer[tempPos + 2] & 0xff) << 16) + | ((buffer[tempPos + 3] & 0xff) << 24)); + } + + @Override + public long readRawLittleEndian64() throws IOException { + int tempPos = pos; + + if (limit - tempPos < FIXED_64_SIZE) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + + final byte[] buffer = this.buffer; + pos = tempPos + FIXED_64_SIZE; + return (((buffer[tempPos] & 0xffL)) + | ((buffer[tempPos + 1] & 0xffL) << 8) + | ((buffer[tempPos + 2] & 0xffL) << 16) + | ((buffer[tempPos + 3] & 0xffL) << 24) + | ((buffer[tempPos + 4] & 0xffL) << 32) + | ((buffer[tempPos + 5] & 0xffL) << 40) + | ((buffer[tempPos + 6] & 0xffL) << 48) + | ((buffer[tempPos + 7] & 0xffL) << 56)); + } + + @Override + public void enableAliasing(boolean enabled) { + this.enableAliasing = enabled; + } + + @Override + public void resetSizeCounter() { + startPos = pos; + } + + @Override + public int pushLimit(int byteLimit) throws InvalidProtocolBufferException { + if (byteLimit < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + byteLimit += getTotalBytesRead(); + final int oldLimit = currentLimit; + if (byteLimit > oldLimit) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + currentLimit = byteLimit; + + recomputeBufferSizeAfterLimit(); + + return oldLimit; + } + + private void recomputeBufferSizeAfterLimit() { + limit += bufferSizeAfterLimit; + final int bufferEnd = limit - startPos; + if (bufferEnd > currentLimit) { + // Limit is in current buffer. + bufferSizeAfterLimit = bufferEnd - currentLimit; + limit -= bufferSizeAfterLimit; + } else { + bufferSizeAfterLimit = 0; + } + } + + @Override + public void popLimit(final int oldLimit) { + currentLimit = oldLimit; + recomputeBufferSizeAfterLimit(); + } + + @Override + public int getBytesUntilLimit() { + if (currentLimit == Integer.MAX_VALUE) { + return -1; + } + + return currentLimit - getTotalBytesRead(); + } + + @Override + public boolean isAtEnd() throws IOException { + return pos == limit; + } + + @Override + public int getTotalBytesRead() { + return pos - startPos; + } + + @Override + public byte readRawByte() throws IOException { + if (pos == limit) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + return buffer[pos++]; + } + + @Override + public byte[] readRawBytes(final int length) throws IOException { + if (length > 0 && length <= (limit - pos)) { + final int tempPos = pos; + pos += length; + return Arrays.copyOfRange(buffer, tempPos, pos); + } + + if (length <= 0) { + if (length == 0) { + return Internal.EMPTY_BYTE_ARRAY; + } else { + throw InvalidProtocolBufferException.negativeSize(); + } + } + throw InvalidProtocolBufferException.truncatedMessage(); + } + + @Override + public void skipRawBytes(final int length) throws IOException { + if (length >= 0 && length <= (limit - pos)) { + // We have all the bytes we need already. + pos += length; + return; + } + + if (length < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } throw InvalidProtocolBufferException.truncatedMessage(); } } /** - * Tries to read more bytes from the input, making at least {@code n} bytes - * available in the buffer. Caller must ensure that the requested space is - * not yet available, and that the requested space is less than BUFFER_SIZE. - * - * @return {@code true} if the bytes could be made available; {@code false} - * if the end of the stream or the current limit was reached. + * A {@link CodedInputStream} implementation that uses a backing direct ByteBuffer as the input. + * Requires the use of {@code sun.misc.Unsafe} to perform fast reads on the buffer. */ - private boolean tryRefillBuffer(int n) throws IOException { - if (bufferPos + n <= bufferSize) { - throw new IllegalStateException( - "refillBuffer() called when " + n + - " bytes were already available in buffer"); + private static final class UnsafeDirectNioDecoder extends CodedInputStream { + /** The direct buffer that is backing this stream. */ + private final ByteBuffer buffer; + + /** + * If {@code true}, indicates that the buffer is backing a {@link ByteString} and is therefore + * considered to be an immutable input source. + */ + private final boolean immutable; + + /** The unsafe address of the content of {@link #buffer}. */ + private final long address; + + /** The unsafe address of the current read limit of the buffer. */ + private long limit; + + /** The unsafe address of the current read position of the buffer. */ + private long pos; + + /** The unsafe address of the starting read position. */ + private long startPos; + + /** The amount of available data in the buffer beyond {@link #limit}. */ + private int bufferSizeAfterLimit; + + /** The last tag that was read from this stream. */ + private int lastTag; + + /** + * If {@code true}, indicates that calls to read {@link ByteString} or {@code byte[]} + * <strong>may</strong> return slices of the underlying buffer, rather than copies. + */ + private boolean enableAliasing; + + /** The absolute position of the end of the current message. */ + private int currentLimit = Integer.MAX_VALUE; + + static boolean isSupported() { + return UnsafeUtil.hasUnsafeByteBufferOperations(); } - if (totalBytesRetired + bufferPos + n > currentLimit) { - // Oops, we hit a limit. - return false; + private UnsafeDirectNioDecoder(ByteBuffer buffer, boolean immutable) { + this.buffer = buffer; + address = UnsafeUtil.addressOffset(buffer); + limit = address + buffer.limit(); + pos = address + buffer.position(); + startPos = pos; + this.immutable = immutable; } - if (refillCallback != null) { - refillCallback.onRefill(); + @Override + public int readTag() throws IOException { + if (isAtEnd()) { + lastTag = 0; + return 0; + } + + lastTag = readRawVarint32(); + if (WireFormat.getTagFieldNumber(lastTag) == 0) { + // If we actually read zero (or any tag number corresponding to field + // number zero), that's not a valid tag. + throw InvalidProtocolBufferException.invalidTag(); + } + return lastTag; } - if (input != null) { - int pos = bufferPos; - if (pos > 0) { - if (bufferSize > pos) { - System.arraycopy(buffer, pos, buffer, 0, bufferSize - pos); - } - totalBytesRetired += pos; - bufferSize -= pos; - bufferPos = 0; + @Override + public void checkLastTagWas(final int value) throws InvalidProtocolBufferException { + if (lastTag != value) { + throw InvalidProtocolBufferException.invalidEndTag(); } + } - int bytesRead = input.read(buffer, bufferSize, buffer.length - bufferSize); - if (bytesRead == 0 || bytesRead < -1 || bytesRead > buffer.length) { - throw new IllegalStateException( - "InputStream#read(byte[]) returned invalid result: " + bytesRead + - "\nThe InputStream implementation is buggy."); + @Override + public int getLastTag() { + return lastTag; + } + + @Override + public boolean skipField(final int tag) throws IOException { + switch (WireFormat.getTagWireType(tag)) { + case WireFormat.WIRETYPE_VARINT: + skipRawVarint(); + return true; + case WireFormat.WIRETYPE_FIXED64: + skipRawBytes(FIXED_64_SIZE); + return true; + case WireFormat.WIRETYPE_LENGTH_DELIMITED: + skipRawBytes(readRawVarint32()); + return true; + case WireFormat.WIRETYPE_START_GROUP: + skipMessage(); + checkLastTagWas( + WireFormat.makeTag(WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP)); + return true; + case WireFormat.WIRETYPE_END_GROUP: + return false; + case WireFormat.WIRETYPE_FIXED32: + skipRawBytes(FIXED_32_SIZE); + return true; + default: + throw InvalidProtocolBufferException.invalidWireType(); } - if (bytesRead > 0) { - bufferSize += bytesRead; - // Integer-overflow-conscious check against sizeLimit - if (totalBytesRetired + n - sizeLimit > 0) { - throw InvalidProtocolBufferException.sizeLimitExceeded(); + } + + @Override + public boolean skipField(final int tag, final CodedOutputStream output) throws IOException { + switch (WireFormat.getTagWireType(tag)) { + case WireFormat.WIRETYPE_VARINT: + { + long value = readInt64(); + output.writeRawVarint32(tag); + output.writeUInt64NoTag(value); + return true; + } + case WireFormat.WIRETYPE_FIXED64: + { + long value = readRawLittleEndian64(); + output.writeRawVarint32(tag); + output.writeFixed64NoTag(value); + return true; + } + case WireFormat.WIRETYPE_LENGTH_DELIMITED: + { + ByteString value = readBytes(); + output.writeRawVarint32(tag); + output.writeBytesNoTag(value); + return true; + } + case WireFormat.WIRETYPE_START_GROUP: + { + output.writeRawVarint32(tag); + skipMessage(output); + int endtag = + WireFormat.makeTag( + WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP); + checkLastTagWas(endtag); + output.writeRawVarint32(endtag); + return true; + } + case WireFormat.WIRETYPE_END_GROUP: + { + return false; + } + case WireFormat.WIRETYPE_FIXED32: + { + int value = readRawLittleEndian32(); + output.writeRawVarint32(tag); + output.writeFixed32NoTag(value); + return true; + } + default: + throw InvalidProtocolBufferException.invalidWireType(); + } + } + + @Override + public void skipMessage() throws IOException { + while (true) { + final int tag = readTag(); + if (tag == 0 || !skipField(tag)) { + return; } - recomputeBufferSizeAfterLimit(); - return (bufferSize >= n) ? true : tryRefillBuffer(n); } } - return false; - } + @Override + public void skipMessage(CodedOutputStream output) throws IOException { + while (true) { + final int tag = readTag(); + if (tag == 0 || !skipField(tag, output)) { + return; + } + } + } - /** - * Read one byte from the input. - * - * @throws InvalidProtocolBufferException The end of the stream or the current - * limit was reached. - */ - public byte readRawByte() throws IOException { - if (bufferPos == bufferSize) { - refillBuffer(1); + + // ----------------------------------------------------------------- + + @Override + public double readDouble() throws IOException { + return Double.longBitsToDouble(readRawLittleEndian64()); } - return buffer[bufferPos++]; - } - /** - * Read a fixed size of bytes from the input. - * - * @throws InvalidProtocolBufferException The end of the stream or the current - * limit was reached. - */ - public byte[] readRawBytes(final int size) throws IOException { - final int pos = bufferPos; - if (size <= (bufferSize - pos) && size > 0) { - bufferPos = pos + size; - return Arrays.copyOfRange(buffer, pos, pos + size); - } else { - return readRawBytesSlowPath(size); + @Override + public float readFloat() throws IOException { + return Float.intBitsToFloat(readRawLittleEndian32()); } - } - /** - * Exactly like readRawBytes, but caller must have already checked the fast - * path: (size <= (bufferSize - pos) && size > 0) - */ - private byte[] readRawBytesSlowPath(final int size) throws IOException { - if (size <= 0) { + @Override + public long readUInt64() throws IOException { + return readRawVarint64(); + } + + @Override + public long readInt64() throws IOException { + return readRawVarint64(); + } + + @Override + public int readInt32() throws IOException { + return readRawVarint32(); + } + + @Override + public long readFixed64() throws IOException { + return readRawLittleEndian64(); + } + + @Override + public int readFixed32() throws IOException { + return readRawLittleEndian32(); + } + + @Override + public boolean readBool() throws IOException { + return readRawVarint64() != 0; + } + + @Override + public String readString() throws IOException { + final int size = readRawVarint32(); + if (size > 0 && size <= remaining()) { + // TODO(nathanmittler): Is there a way to avoid this copy? + byte[] bytes = copyToArray(pos, pos + size); + String result = new String(bytes, UTF_8); + pos += size; + return result; + } + if (size == 0) { - return Internal.EMPTY_BYTE_ARRAY; - } else { + return ""; + } + if (size < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + throw InvalidProtocolBufferException.truncatedMessage(); + } + + @Override + public String readStringRequireUtf8() throws IOException { + final int size = readRawVarint32(); + if (size >= 0 && size <= remaining()) { + // TODO(nathanmittler): Is there a way to avoid this copy? + byte[] bytes = copyToArray(pos, pos + size); + // TODO(martinrb): We could save a pass by validating while decoding. + if (!Utf8.isValidUtf8(bytes)) { + throw InvalidProtocolBufferException.invalidUtf8(); + } + + String result = new String(bytes, UTF_8); + pos += size; + return result; + } + + if (size == 0) { + return ""; + } + if (size <= 0) { throw InvalidProtocolBufferException.negativeSize(); } + throw InvalidProtocolBufferException.truncatedMessage(); + } + + @Override + public void readGroup( + final int fieldNumber, + final MessageLite.Builder builder, + final ExtensionRegistryLite extensionRegistry) + throws IOException { + if (recursionDepth >= recursionLimit) { + throw InvalidProtocolBufferException.recursionLimitExceeded(); + } + ++recursionDepth; + builder.mergeFrom(this, extensionRegistry); + checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); + --recursionDepth; } - // Verify that the message size so far has not exceeded sizeLimit. - int currentMessageSize = totalBytesRetired + bufferPos + size; - if (currentMessageSize > sizeLimit) { - throw InvalidProtocolBufferException.sizeLimitExceeded(); + + @Override + public <T extends MessageLite> T readGroup( + final int fieldNumber, + final Parser<T> parser, + final ExtensionRegistryLite extensionRegistry) + throws IOException { + if (recursionDepth >= recursionLimit) { + throw InvalidProtocolBufferException.recursionLimitExceeded(); + } + ++recursionDepth; + T result = parser.parsePartialFrom(this, extensionRegistry); + checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); + --recursionDepth; + return result; } - // Verify that the message size so far has not exceeded currentLimit. - if (currentMessageSize > currentLimit) { - // Read to the end of the stream anyway. - skipRawBytes(currentLimit - totalBytesRetired - bufferPos); + @Deprecated + @Override + public void readUnknownGroup(final int fieldNumber, final MessageLite.Builder builder) + throws IOException { + readGroup(fieldNumber, builder, ExtensionRegistryLite.getEmptyRegistry()); + } + + @Override + public void readMessage( + final MessageLite.Builder builder, final ExtensionRegistryLite extensionRegistry) + throws IOException { + final int length = readRawVarint32(); + if (recursionDepth >= recursionLimit) { + throw InvalidProtocolBufferException.recursionLimitExceeded(); + } + final int oldLimit = pushLimit(length); + ++recursionDepth; + builder.mergeFrom(this, extensionRegistry); + checkLastTagWas(0); + --recursionDepth; + popLimit(oldLimit); + } + + + @Override + public <T extends MessageLite> T readMessage( + final Parser<T> parser, final ExtensionRegistryLite extensionRegistry) throws IOException { + int length = readRawVarint32(); + if (recursionDepth >= recursionLimit) { + throw InvalidProtocolBufferException.recursionLimitExceeded(); + } + final int oldLimit = pushLimit(length); + ++recursionDepth; + T result = parser.parsePartialFrom(this, extensionRegistry); + checkLastTagWas(0); + --recursionDepth; + popLimit(oldLimit); + return result; + } + + @Override + public ByteString readBytes() throws IOException { + final int size = readRawVarint32(); + if (size > 0 && size <= remaining()) { + ByteBuffer result; + if (immutable && enableAliasing) { + result = slice(pos, pos + size); + } else { + result = copy(pos, pos + size); + } + pos += size; + return ByteString.wrap(result); + } + + if (size == 0) { + return ByteString.EMPTY; + } + if (size < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } throw InvalidProtocolBufferException.truncatedMessage(); } - // We need the input stream to proceed. - if (input == null) { + @Override + public byte[] readByteArray() throws IOException { + return readRawBytes(readRawVarint32()); + } + + @Override + public ByteBuffer readByteBuffer() throws IOException { + final int size = readRawVarint32(); + if (size > 0 && size <= remaining()) { + ByteBuffer result; + // "Immutable" implies that buffer is backing a ByteString. + // Disallow slicing in this case to prevent the caller from modifying the contents + // of the ByteString. + if (!immutable && enableAliasing) { + result = slice(pos, pos + size); + } else { + result = copy(pos, pos + size); + } + pos += size; + // TODO(nathanmittler): Investigate making the ByteBuffer be made read-only + return result; + } + + if (size == 0) { + return EMPTY_BYTE_BUFFER; + } + if (size < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } throw InvalidProtocolBufferException.truncatedMessage(); } - final int originalBufferPos = bufferPos; - final int bufferedBytes = bufferSize - bufferPos; + @Override + public int readUInt32() throws IOException { + return readRawVarint32(); + } + + @Override + public int readEnum() throws IOException { + return readRawVarint32(); + } - // Mark the current buffer consumed. - totalBytesRetired += bufferSize; - bufferPos = 0; - bufferSize = 0; + @Override + public int readSFixed32() throws IOException { + return readRawLittleEndian32(); + } - // Determine the number of bytes we need to read from the input stream. - int sizeLeft = size - bufferedBytes; - // TODO(nathanmittler): Consider using a value larger than BUFFER_SIZE. - if (sizeLeft < BUFFER_SIZE || sizeLeft <= input.available()) { - // Either the bytes we need are known to be available, or the required buffer is - // within an allowed threshold - go ahead and allocate the buffer now. - final byte[] bytes = new byte[size]; + @Override + public long readSFixed64() throws IOException { + return readRawLittleEndian64(); + } - // Copy all of the buffered bytes to the result buffer. - System.arraycopy(buffer, originalBufferPos, bytes, 0, bufferedBytes); + @Override + public int readSInt32() throws IOException { + return decodeZigZag32(readRawVarint32()); + } + + @Override + public long readSInt64() throws IOException { + return decodeZigZag64(readRawVarint64()); + } + + // ================================================================= - // Fill the remaining bytes from the input stream. - int pos = bufferedBytes; - while (pos < bytes.length) { - int n = input.read(bytes, pos, size - pos); - if (n == -1) { - throw InvalidProtocolBufferException.truncatedMessage(); + @Override + public int readRawVarint32() throws IOException { + // See implementation notes for readRawVarint64 + fastpath: + { + long tempPos = pos; + + if (limit == tempPos) { + break fastpath; + } + + int x; + if ((x = UnsafeUtil.getByte(tempPos++)) >= 0) { + pos = tempPos; + return x; + } else if (limit - tempPos < 9) { + break fastpath; + } else if ((x ^= (UnsafeUtil.getByte(tempPos++) << 7)) < 0) { + x ^= (~0 << 7); + } else if ((x ^= (UnsafeUtil.getByte(tempPos++) << 14)) >= 0) { + x ^= (~0 << 7) ^ (~0 << 14); + } else if ((x ^= (UnsafeUtil.getByte(tempPos++) << 21)) < 0) { + x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21); + } else { + int y = UnsafeUtil.getByte(tempPos++); + x ^= y << 28; + x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21) ^ (~0 << 28); + if (y < 0 + && UnsafeUtil.getByte(tempPos++) < 0 + && UnsafeUtil.getByte(tempPos++) < 0 + && UnsafeUtil.getByte(tempPos++) < 0 + && UnsafeUtil.getByte(tempPos++) < 0 + && UnsafeUtil.getByte(tempPos++) < 0) { + break fastpath; // Will throw malformedVarint() + } } - totalBytesRetired += n; - pos += n; + pos = tempPos; + return x; } + return (int) readRawVarint64SlowPath(); + } - return bytes; + private void skipRawVarint() throws IOException { + if (remaining() >= MAX_VARINT_SIZE) { + skipRawVarintFastPath(); + } else { + skipRawVarintSlowPath(); + } } - // The size is very large. For security reasons, we can't allocate the - // entire byte array yet. The size comes directly from the input, so a - // maliciously-crafted message could provide a bogus very large size in - // order to trick the app into allocating a lot of memory. We avoid this - // by allocating and reading only a small chunk at a time, so that the - // malicious message must actually *be* extremely large to cause - // problems. Meanwhile, we limit the allowed size of a message elsewhere. - final List<byte[]> chunks = new ArrayList<byte[]>(); - - while (sizeLeft > 0) { - // TODO(nathanmittler): Consider using a value larger than BUFFER_SIZE. - final byte[] chunk = new byte[Math.min(sizeLeft, BUFFER_SIZE)]; - int pos = 0; - while (pos < chunk.length) { - final int n = input.read(chunk, pos, chunk.length - pos); - if (n == -1) { - throw InvalidProtocolBufferException.truncatedMessage(); + private void skipRawVarintFastPath() throws IOException { + for (int i = 0; i < MAX_VARINT_SIZE; i++) { + if (UnsafeUtil.getByte(pos++) >= 0) { + return; } - totalBytesRetired += n; - pos += n; } - sizeLeft -= chunk.length; - chunks.add(chunk); + throw InvalidProtocolBufferException.malformedVarint(); } - // OK, got everything. Now concatenate it all into one buffer. - final byte[] bytes = new byte[size]; + private void skipRawVarintSlowPath() throws IOException { + for (int i = 0; i < MAX_VARINT_SIZE; i++) { + if (readRawByte() >= 0) { + return; + } + } + throw InvalidProtocolBufferException.malformedVarint(); + } - // Start by copying the leftover bytes from this.buffer. - System.arraycopy(buffer, originalBufferPos, bytes, 0, bufferedBytes); + @Override + public long readRawVarint64() throws IOException { + // Implementation notes: + // + // Optimized for one-byte values, expected to be common. + // The particular code below was selected from various candidates + // empirically, by winning VarintBenchmark. + // + // Sign extension of (signed) Java bytes is usually a nuisance, but + // we exploit it here to more easily obtain the sign of bytes read. + // Instead of cleaning up the sign extension bits by masking eagerly, + // we delay until we find the final (positive) byte, when we clear all + // accumulated bits with one xor. We depend on javac to constant fold. + fastpath: + { + long tempPos = pos; + + if (limit == tempPos) { + break fastpath; + } - // And now all the chunks. - int pos = bufferedBytes; - for (final byte[] chunk : chunks) { - System.arraycopy(chunk, 0, bytes, pos, chunk.length); - pos += chunk.length; + long x; + int y; + if ((y = UnsafeUtil.getByte(tempPos++)) >= 0) { + pos = tempPos; + return y; + } else if (limit - tempPos < 9) { + break fastpath; + } else if ((y ^= (UnsafeUtil.getByte(tempPos++) << 7)) < 0) { + x = y ^ (~0 << 7); + } else if ((y ^= (UnsafeUtil.getByte(tempPos++) << 14)) >= 0) { + x = y ^ ((~0 << 7) ^ (~0 << 14)); + } else if ((y ^= (UnsafeUtil.getByte(tempPos++) << 21)) < 0) { + x = y ^ ((~0 << 7) ^ (~0 << 14) ^ (~0 << 21)); + } else if ((x = y ^ ((long) UnsafeUtil.getByte(tempPos++) << 28)) >= 0L) { + x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28); + } else if ((x ^= ((long) UnsafeUtil.getByte(tempPos++) << 35)) < 0L) { + x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35); + } else if ((x ^= ((long) UnsafeUtil.getByte(tempPos++) << 42)) >= 0L) { + x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35) ^ (~0L << 42); + } else if ((x ^= ((long) UnsafeUtil.getByte(tempPos++) << 49)) < 0L) { + x ^= + (~0L << 7) + ^ (~0L << 14) + ^ (~0L << 21) + ^ (~0L << 28) + ^ (~0L << 35) + ^ (~0L << 42) + ^ (~0L << 49); + } else { + x ^= ((long) UnsafeUtil.getByte(tempPos++) << 56); + x ^= + (~0L << 7) + ^ (~0L << 14) + ^ (~0L << 21) + ^ (~0L << 28) + ^ (~0L << 35) + ^ (~0L << 42) + ^ (~0L << 49) + ^ (~0L << 56); + if (x < 0L) { + if (UnsafeUtil.getByte(tempPos++) < 0L) { + break fastpath; // Will throw malformedVarint() + } + } + } + pos = tempPos; + return x; + } + return readRawVarint64SlowPath(); } - // Done. - return bytes; - } + @Override + long readRawVarint64SlowPath() throws IOException { + long result = 0; + for (int shift = 0; shift < 64; shift += 7) { + final byte b = readRawByte(); + result |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return result; + } + } + throw InvalidProtocolBufferException.malformedVarint(); + } - /** - * Reads and discards {@code size} bytes. - * - * @throws InvalidProtocolBufferException The end of the stream or the current - * limit was reached. - */ - public void skipRawBytes(final int size) throws IOException { - if (size <= (bufferSize - bufferPos) && size >= 0) { - // We have all the bytes we need already. - bufferPos += size; - } else { - skipRawBytesSlowPath(size); + @Override + public int readRawLittleEndian32() throws IOException { + long tempPos = pos; + + if (limit - tempPos < FIXED_32_SIZE) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + + pos = tempPos + FIXED_32_SIZE; + return (((UnsafeUtil.getByte(tempPos) & 0xff)) + | ((UnsafeUtil.getByte(tempPos + 1) & 0xff) << 8) + | ((UnsafeUtil.getByte(tempPos + 2) & 0xff) << 16) + | ((UnsafeUtil.getByte(tempPos + 3) & 0xff) << 24)); + } + + @Override + public long readRawLittleEndian64() throws IOException { + long tempPos = pos; + + if (limit - tempPos < FIXED_64_SIZE) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + + pos = tempPos + FIXED_64_SIZE; + return (((UnsafeUtil.getByte(tempPos) & 0xffL)) + | ((UnsafeUtil.getByte(tempPos + 1) & 0xffL) << 8) + | ((UnsafeUtil.getByte(tempPos + 2) & 0xffL) << 16) + | ((UnsafeUtil.getByte(tempPos + 3) & 0xffL) << 24) + | ((UnsafeUtil.getByte(tempPos + 4) & 0xffL) << 32) + | ((UnsafeUtil.getByte(tempPos + 5) & 0xffL) << 40) + | ((UnsafeUtil.getByte(tempPos + 6) & 0xffL) << 48) + | ((UnsafeUtil.getByte(tempPos + 7) & 0xffL) << 56)); + } + + @Override + public void enableAliasing(boolean enabled) { + this.enableAliasing = enabled; + } + + @Override + public void resetSizeCounter() { + startPos = pos; + } + + @Override + public int pushLimit(int byteLimit) throws InvalidProtocolBufferException { + if (byteLimit < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + byteLimit += getTotalBytesRead(); + final int oldLimit = currentLimit; + if (byteLimit > oldLimit) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + currentLimit = byteLimit; + + recomputeBufferSizeAfterLimit(); + + return oldLimit; + } + + @Override + public void popLimit(final int oldLimit) { + currentLimit = oldLimit; + recomputeBufferSizeAfterLimit(); + } + + @Override + public int getBytesUntilLimit() { + if (currentLimit == Integer.MAX_VALUE) { + return -1; + } + + return currentLimit - getTotalBytesRead(); + } + + @Override + public boolean isAtEnd() throws IOException { + return pos == limit; + } + + @Override + public int getTotalBytesRead() { + return (int) (pos - startPos); + } + + @Override + public byte readRawByte() throws IOException { + if (pos == limit) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + return UnsafeUtil.getByte(pos++); + } + + @Override + public byte[] readRawBytes(final int length) throws IOException { + if (length >= 0 && length <= remaining()) { + byte[] bytes = new byte[length]; + slice(pos, pos + length).get(bytes); + pos += length; + return bytes; + } + + if (length <= 0) { + if (length == 0) { + return EMPTY_BYTE_ARRAY; + } else { + throw InvalidProtocolBufferException.negativeSize(); + } + } + + throw InvalidProtocolBufferException.truncatedMessage(); + } + + @Override + public void skipRawBytes(final int length) throws IOException { + if (length >= 0 && length <= remaining()) { + // We have all the bytes we need already. + pos += length; + return; + } + + if (length < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + throw InvalidProtocolBufferException.truncatedMessage(); + } + + private void recomputeBufferSizeAfterLimit() { + limit += bufferSizeAfterLimit; + final int bufferEnd = (int) (limit - startPos); + if (bufferEnd > currentLimit) { + // Limit is in current buffer. + bufferSizeAfterLimit = bufferEnd - currentLimit; + limit -= bufferSizeAfterLimit; + } else { + bufferSizeAfterLimit = 0; + } + } + + private int remaining() { + return (int) (limit - pos); + } + + private int bufferPos(long pos) { + return (int) (pos - address); + } + + private ByteBuffer slice(long begin, long end) throws IOException { + int prevPos = buffer.position(); + int prevLimit = buffer.limit(); + try { + buffer.position(bufferPos(begin)); + buffer.limit(bufferPos(end)); + return buffer.slice(); + } catch (IllegalArgumentException e) { + throw InvalidProtocolBufferException.truncatedMessage(); + } finally { + buffer.position(prevPos); + buffer.limit(prevLimit); + } + } + + private ByteBuffer copy(long begin, long end) throws IOException { + return ByteBuffer.wrap(copyToArray(begin, end)); + } + + private byte[] copyToArray(long begin, long end) throws IOException { + int prevPos = buffer.position(); + int prevLimit = buffer.limit(); + try { + buffer.position(bufferPos(begin)); + buffer.limit(bufferPos(end)); + byte[] bytes = new byte[(int) (end - begin)]; + buffer.get(bytes); + return bytes; + } catch (IllegalArgumentException e) { + throw InvalidProtocolBufferException.truncatedMessage(); + } finally { + buffer.position(prevPos); + buffer.limit(prevLimit); + } } } /** - * Exactly like skipRawBytes, but caller must have already checked the fast - * path: (size <= (bufferSize - pos) && size >= 0) + * Implementation of {@link CodedInputStream} that uses an {@link InputStream} as the data source. */ - private void skipRawBytesSlowPath(final int size) throws IOException { - if (size < 0) { - throw InvalidProtocolBufferException.negativeSize(); + private static final class StreamDecoder extends CodedInputStream { + private final InputStream input; + private final byte[] buffer; + /** bufferSize represents how many bytes are currently filled in the buffer */ + private int bufferSize; + + private int bufferSizeAfterLimit; + private int pos; + private int lastTag; + + /** + * The total number of bytes read before the current buffer. The total bytes read up to the + * current position can be computed as {@code totalBytesRetired + pos}. This value may be + * negative if reading started in the middle of the current buffer (e.g. if the constructor that + * takes a byte array and an offset was used). + */ + private int totalBytesRetired; + + /** The absolute position of the end of the current message. */ + private int currentLimit = Integer.MAX_VALUE; + + private StreamDecoder(final InputStream input, int bufferSize) { + checkNotNull(input, "input"); + this.input = input; + this.buffer = new byte[bufferSize]; + this.bufferSize = 0; + pos = 0; + totalBytesRetired = 0; } - if (totalBytesRetired + bufferPos + size > currentLimit) { - // Read to the end of the stream anyway. - skipRawBytes(currentLimit - totalBytesRetired - bufferPos); - // Then fail. - throw InvalidProtocolBufferException.truncatedMessage(); + @Override + public int readTag() throws IOException { + if (isAtEnd()) { + lastTag = 0; + return 0; + } + + lastTag = readRawVarint32(); + if (WireFormat.getTagFieldNumber(lastTag) == 0) { + // If we actually read zero (or any tag number corresponding to field + // number zero), that's not a valid tag. + throw InvalidProtocolBufferException.invalidTag(); + } + return lastTag; } - // Skipping more bytes than are in the buffer. First skip what we have. - int pos = bufferSize - bufferPos; - bufferPos = bufferSize; + @Override + public void checkLastTagWas(final int value) throws InvalidProtocolBufferException { + if (lastTag != value) { + throw InvalidProtocolBufferException.invalidEndTag(); + } + } - // Keep refilling the buffer until we get to the point we wanted to skip to. - // This has the side effect of ensuring the limits are updated correctly. - refillBuffer(1); - while (size - pos > bufferSize) { - pos += bufferSize; - bufferPos = bufferSize; - refillBuffer(1); + @Override + public int getLastTag() { + return lastTag; + } + + @Override + public boolean skipField(final int tag) throws IOException { + switch (WireFormat.getTagWireType(tag)) { + case WireFormat.WIRETYPE_VARINT: + skipRawVarint(); + return true; + case WireFormat.WIRETYPE_FIXED64: + skipRawBytes(FIXED_64_SIZE); + return true; + case WireFormat.WIRETYPE_LENGTH_DELIMITED: + skipRawBytes(readRawVarint32()); + return true; + case WireFormat.WIRETYPE_START_GROUP: + skipMessage(); + checkLastTagWas( + WireFormat.makeTag(WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP)); + return true; + case WireFormat.WIRETYPE_END_GROUP: + return false; + case WireFormat.WIRETYPE_FIXED32: + skipRawBytes(FIXED_32_SIZE); + return true; + default: + throw InvalidProtocolBufferException.invalidWireType(); + } + } + + @Override + public boolean skipField(final int tag, final CodedOutputStream output) throws IOException { + switch (WireFormat.getTagWireType(tag)) { + case WireFormat.WIRETYPE_VARINT: + { + long value = readInt64(); + output.writeRawVarint32(tag); + output.writeUInt64NoTag(value); + return true; + } + case WireFormat.WIRETYPE_FIXED64: + { + long value = readRawLittleEndian64(); + output.writeRawVarint32(tag); + output.writeFixed64NoTag(value); + return true; + } + case WireFormat.WIRETYPE_LENGTH_DELIMITED: + { + ByteString value = readBytes(); + output.writeRawVarint32(tag); + output.writeBytesNoTag(value); + return true; + } + case WireFormat.WIRETYPE_START_GROUP: + { + output.writeRawVarint32(tag); + skipMessage(output); + int endtag = + WireFormat.makeTag( + WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP); + checkLastTagWas(endtag); + output.writeRawVarint32(endtag); + return true; + } + case WireFormat.WIRETYPE_END_GROUP: + { + return false; + } + case WireFormat.WIRETYPE_FIXED32: + { + int value = readRawLittleEndian32(); + output.writeRawVarint32(tag); + output.writeFixed32NoTag(value); + return true; + } + default: + throw InvalidProtocolBufferException.invalidWireType(); + } + } + + @Override + public void skipMessage() throws IOException { + while (true) { + final int tag = readTag(); + if (tag == 0 || !skipField(tag)) { + return; + } + } + } + + @Override + public void skipMessage(CodedOutputStream output) throws IOException { + while (true) { + final int tag = readTag(); + if (tag == 0 || !skipField(tag, output)) { + return; + } + } + } + + /** Collects the bytes skipped and returns the data in a ByteBuffer. */ + private class SkippedDataSink implements RefillCallback { + private int lastPos = pos; + private ByteArrayOutputStream byteArrayStream; + + @Override + public void onRefill() { + if (byteArrayStream == null) { + byteArrayStream = new ByteArrayOutputStream(); + } + byteArrayStream.write(buffer, lastPos, pos - lastPos); + lastPos = 0; + } + + /** Gets skipped data in a ByteBuffer. This method should only be called once. */ + ByteBuffer getSkippedData() { + if (byteArrayStream == null) { + return ByteBuffer.wrap(buffer, lastPos, pos - lastPos); + } else { + byteArrayStream.write(buffer, lastPos, pos); + return ByteBuffer.wrap(byteArrayStream.toByteArray()); + } + } + } + + + // ----------------------------------------------------------------- + + @Override + public double readDouble() throws IOException { + return Double.longBitsToDouble(readRawLittleEndian64()); + } + + @Override + public float readFloat() throws IOException { + return Float.intBitsToFloat(readRawLittleEndian32()); + } + + @Override + public long readUInt64() throws IOException { + return readRawVarint64(); + } + + @Override + public long readInt64() throws IOException { + return readRawVarint64(); + } + + @Override + public int readInt32() throws IOException { + return readRawVarint32(); + } + + @Override + public long readFixed64() throws IOException { + return readRawLittleEndian64(); + } + + @Override + public int readFixed32() throws IOException { + return readRawLittleEndian32(); } - bufferPos = size - pos; + @Override + public boolean readBool() throws IOException { + return readRawVarint64() != 0; + } + + @Override + public String readString() throws IOException { + final int size = readRawVarint32(); + if (size > 0 && size <= (bufferSize - pos)) { + // Fast path: We already have the bytes in a contiguous buffer, so + // just copy directly from it. + final String result = new String(buffer, pos, size, UTF_8); + pos += size; + return result; + } + if (size == 0) { + return ""; + } + if (size <= bufferSize) { + refillBuffer(size); + String result = new String(buffer, pos, size, UTF_8); + pos += size; + return result; + } + // Slow path: Build a byte array first then copy it. + return new String(readRawBytesSlowPath(size), UTF_8); + } + + @Override + public String readStringRequireUtf8() throws IOException { + final int size = readRawVarint32(); + final byte[] bytes; + final int oldPos = pos; + final int tempPos; + if (size <= (bufferSize - oldPos) && size > 0) { + // Fast path: We already have the bytes in a contiguous buffer, so + // just copy directly from it. + bytes = buffer; + pos = oldPos + size; + tempPos = oldPos; + } else if (size == 0) { + return ""; + } else if (size <= bufferSize) { + refillBuffer(size); + bytes = buffer; + tempPos = 0; + pos = tempPos + size; + } else { + // Slow path: Build a byte array first then copy it. + bytes = readRawBytesSlowPath(size); + tempPos = 0; + } + // TODO(martinrb): We could save a pass by validating while decoding. + if (!Utf8.isValidUtf8(bytes, tempPos, tempPos + size)) { + throw InvalidProtocolBufferException.invalidUtf8(); + } + return new String(bytes, tempPos, size, UTF_8); + } + + @Override + public void readGroup( + final int fieldNumber, + final MessageLite.Builder builder, + final ExtensionRegistryLite extensionRegistry) + throws IOException { + if (recursionDepth >= recursionLimit) { + throw InvalidProtocolBufferException.recursionLimitExceeded(); + } + ++recursionDepth; + builder.mergeFrom(this, extensionRegistry); + checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); + --recursionDepth; + } + + + @Override + public <T extends MessageLite> T readGroup( + final int fieldNumber, + final Parser<T> parser, + final ExtensionRegistryLite extensionRegistry) + throws IOException { + if (recursionDepth >= recursionLimit) { + throw InvalidProtocolBufferException.recursionLimitExceeded(); + } + ++recursionDepth; + T result = parser.parsePartialFrom(this, extensionRegistry); + checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); + --recursionDepth; + return result; + } + + @Deprecated + @Override + public void readUnknownGroup(final int fieldNumber, final MessageLite.Builder builder) + throws IOException { + readGroup(fieldNumber, builder, ExtensionRegistryLite.getEmptyRegistry()); + } + + @Override + public void readMessage( + final MessageLite.Builder builder, final ExtensionRegistryLite extensionRegistry) + throws IOException { + final int length = readRawVarint32(); + if (recursionDepth >= recursionLimit) { + throw InvalidProtocolBufferException.recursionLimitExceeded(); + } + final int oldLimit = pushLimit(length); + ++recursionDepth; + builder.mergeFrom(this, extensionRegistry); + checkLastTagWas(0); + --recursionDepth; + popLimit(oldLimit); + } + + + @Override + public <T extends MessageLite> T readMessage( + final Parser<T> parser, final ExtensionRegistryLite extensionRegistry) throws IOException { + int length = readRawVarint32(); + if (recursionDepth >= recursionLimit) { + throw InvalidProtocolBufferException.recursionLimitExceeded(); + } + final int oldLimit = pushLimit(length); + ++recursionDepth; + T result = parser.parsePartialFrom(this, extensionRegistry); + checkLastTagWas(0); + --recursionDepth; + popLimit(oldLimit); + return result; + } + + @Override + public ByteString readBytes() throws IOException { + final int size = readRawVarint32(); + if (size <= (bufferSize - pos) && size > 0) { + // Fast path: We already have the bytes in a contiguous buffer, so + // just copy directly from it. + final ByteString result = ByteString.copyFrom(buffer, pos, size); + pos += size; + return result; + } + if (size == 0) { + return ByteString.EMPTY; + } + // Slow path: Build a byte array first then copy it. + return ByteString.wrap(readRawBytesSlowPath(size)); + } + + @Override + public byte[] readByteArray() throws IOException { + final int size = readRawVarint32(); + if (size <= (bufferSize - pos) && size > 0) { + // Fast path: We already have the bytes in a contiguous buffer, so + // just copy directly from it. + final byte[] result = Arrays.copyOfRange(buffer, pos, pos + size); + pos += size; + return result; + } else { + // Slow path: Build a byte array first then copy it. + return readRawBytesSlowPath(size); + } + } + + @Override + public ByteBuffer readByteBuffer() throws IOException { + final int size = readRawVarint32(); + if (size <= (bufferSize - pos) && size > 0) { + // Fast path: We already have the bytes in a contiguous buffer. + ByteBuffer result = ByteBuffer.wrap(Arrays.copyOfRange(buffer, pos, pos + size)); + pos += size; + return result; + } + if (size == 0) { + return Internal.EMPTY_BYTE_BUFFER; + } + // Slow path: Build a byte array first then copy it. + return ByteBuffer.wrap(readRawBytesSlowPath(size)); + } + + @Override + public int readUInt32() throws IOException { + return readRawVarint32(); + } + + @Override + public int readEnum() throws IOException { + return readRawVarint32(); + } + + @Override + public int readSFixed32() throws IOException { + return readRawLittleEndian32(); + } + + @Override + public long readSFixed64() throws IOException { + return readRawLittleEndian64(); + } + + @Override + public int readSInt32() throws IOException { + return decodeZigZag32(readRawVarint32()); + } + + @Override + public long readSInt64() throws IOException { + return decodeZigZag64(readRawVarint64()); + } + + // ================================================================= + + @Override + public int readRawVarint32() throws IOException { + // See implementation notes for readRawVarint64 + fastpath: + { + int tempPos = pos; + + if (bufferSize == tempPos) { + break fastpath; + } + + final byte[] buffer = this.buffer; + int x; + if ((x = buffer[tempPos++]) >= 0) { + pos = tempPos; + return x; + } else if (bufferSize - tempPos < 9) { + break fastpath; + } else if ((x ^= (buffer[tempPos++] << 7)) < 0) { + x ^= (~0 << 7); + } else if ((x ^= (buffer[tempPos++] << 14)) >= 0) { + x ^= (~0 << 7) ^ (~0 << 14); + } else if ((x ^= (buffer[tempPos++] << 21)) < 0) { + x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21); + } else { + int y = buffer[tempPos++]; + x ^= y << 28; + x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21) ^ (~0 << 28); + if (y < 0 + && buffer[tempPos++] < 0 + && buffer[tempPos++] < 0 + && buffer[tempPos++] < 0 + && buffer[tempPos++] < 0 + && buffer[tempPos++] < 0) { + break fastpath; // Will throw malformedVarint() + } + } + pos = tempPos; + return x; + } + return (int) readRawVarint64SlowPath(); + } + + private void skipRawVarint() throws IOException { + if (bufferSize - pos >= MAX_VARINT_SIZE) { + skipRawVarintFastPath(); + } else { + skipRawVarintSlowPath(); + } + } + + private void skipRawVarintFastPath() throws IOException { + for (int i = 0; i < MAX_VARINT_SIZE; i++) { + if (buffer[pos++] >= 0) { + return; + } + } + throw InvalidProtocolBufferException.malformedVarint(); + } + + private void skipRawVarintSlowPath() throws IOException { + for (int i = 0; i < MAX_VARINT_SIZE; i++) { + if (readRawByte() >= 0) { + return; + } + } + throw InvalidProtocolBufferException.malformedVarint(); + } + + @Override + public long readRawVarint64() throws IOException { + // Implementation notes: + // + // Optimized for one-byte values, expected to be common. + // The particular code below was selected from various candidates + // empirically, by winning VarintBenchmark. + // + // Sign extension of (signed) Java bytes is usually a nuisance, but + // we exploit it here to more easily obtain the sign of bytes read. + // Instead of cleaning up the sign extension bits by masking eagerly, + // we delay until we find the final (positive) byte, when we clear all + // accumulated bits with one xor. We depend on javac to constant fold. + fastpath: + { + int tempPos = pos; + + if (bufferSize == tempPos) { + break fastpath; + } + + final byte[] buffer = this.buffer; + long x; + int y; + if ((y = buffer[tempPos++]) >= 0) { + pos = tempPos; + return y; + } else if (bufferSize - tempPos < 9) { + break fastpath; + } else if ((y ^= (buffer[tempPos++] << 7)) < 0) { + x = y ^ (~0 << 7); + } else if ((y ^= (buffer[tempPos++] << 14)) >= 0) { + x = y ^ ((~0 << 7) ^ (~0 << 14)); + } else if ((y ^= (buffer[tempPos++] << 21)) < 0) { + x = y ^ ((~0 << 7) ^ (~0 << 14) ^ (~0 << 21)); + } else if ((x = y ^ ((long) buffer[tempPos++] << 28)) >= 0L) { + x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28); + } else if ((x ^= ((long) buffer[tempPos++] << 35)) < 0L) { + x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35); + } else if ((x ^= ((long) buffer[tempPos++] << 42)) >= 0L) { + x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35) ^ (~0L << 42); + } else if ((x ^= ((long) buffer[tempPos++] << 49)) < 0L) { + x ^= + (~0L << 7) + ^ (~0L << 14) + ^ (~0L << 21) + ^ (~0L << 28) + ^ (~0L << 35) + ^ (~0L << 42) + ^ (~0L << 49); + } else { + x ^= ((long) buffer[tempPos++] << 56); + x ^= + (~0L << 7) + ^ (~0L << 14) + ^ (~0L << 21) + ^ (~0L << 28) + ^ (~0L << 35) + ^ (~0L << 42) + ^ (~0L << 49) + ^ (~0L << 56); + if (x < 0L) { + if (buffer[tempPos++] < 0L) { + break fastpath; // Will throw malformedVarint() + } + } + } + pos = tempPos; + return x; + } + return readRawVarint64SlowPath(); + } + + @Override + long readRawVarint64SlowPath() throws IOException { + long result = 0; + for (int shift = 0; shift < 64; shift += 7) { + final byte b = readRawByte(); + result |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return result; + } + } + throw InvalidProtocolBufferException.malformedVarint(); + } + + @Override + public int readRawLittleEndian32() throws IOException { + int tempPos = pos; + + if (bufferSize - tempPos < FIXED_32_SIZE) { + refillBuffer(FIXED_32_SIZE); + tempPos = pos; + } + + final byte[] buffer = this.buffer; + pos = tempPos + FIXED_32_SIZE; + return (((buffer[tempPos] & 0xff)) + | ((buffer[tempPos + 1] & 0xff) << 8) + | ((buffer[tempPos + 2] & 0xff) << 16) + | ((buffer[tempPos + 3] & 0xff) << 24)); + } + + @Override + public long readRawLittleEndian64() throws IOException { + int tempPos = pos; + + if (bufferSize - tempPos < FIXED_64_SIZE) { + refillBuffer(FIXED_64_SIZE); + tempPos = pos; + } + + final byte[] buffer = this.buffer; + pos = tempPos + FIXED_64_SIZE; + return (((buffer[tempPos] & 0xffL)) + | ((buffer[tempPos + 1] & 0xffL) << 8) + | ((buffer[tempPos + 2] & 0xffL) << 16) + | ((buffer[tempPos + 3] & 0xffL) << 24) + | ((buffer[tempPos + 4] & 0xffL) << 32) + | ((buffer[tempPos + 5] & 0xffL) << 40) + | ((buffer[tempPos + 6] & 0xffL) << 48) + | ((buffer[tempPos + 7] & 0xffL) << 56)); + } + + // ----------------------------------------------------------------- + + @Override + public void enableAliasing(boolean enabled) { + // TODO(nathanmittler): Ideally we should throw here. Do nothing for backward compatibility. + } + + @Override + public void resetSizeCounter() { + totalBytesRetired = -pos; + } + + @Override + public int pushLimit(int byteLimit) throws InvalidProtocolBufferException { + if (byteLimit < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + byteLimit += totalBytesRetired + pos; + final int oldLimit = currentLimit; + if (byteLimit > oldLimit) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + currentLimit = byteLimit; + + recomputeBufferSizeAfterLimit(); + + return oldLimit; + } + + private void recomputeBufferSizeAfterLimit() { + bufferSize += bufferSizeAfterLimit; + final int bufferEnd = totalBytesRetired + bufferSize; + if (bufferEnd > currentLimit) { + // Limit is in current buffer. + bufferSizeAfterLimit = bufferEnd - currentLimit; + bufferSize -= bufferSizeAfterLimit; + } else { + bufferSizeAfterLimit = 0; + } + } + + @Override + public void popLimit(final int oldLimit) { + currentLimit = oldLimit; + recomputeBufferSizeAfterLimit(); + } + + @Override + public int getBytesUntilLimit() { + if (currentLimit == Integer.MAX_VALUE) { + return -1; + } + + final int currentAbsolutePosition = totalBytesRetired + pos; + return currentLimit - currentAbsolutePosition; + } + + @Override + public boolean isAtEnd() throws IOException { + return pos == bufferSize && !tryRefillBuffer(1); + } + + @Override + public int getTotalBytesRead() { + return totalBytesRetired + pos; + } + + private interface RefillCallback { + void onRefill(); + } + + private RefillCallback refillCallback = null; + + /** + * Reads more bytes from the input, making at least {@code n} bytes available in the buffer. + * Caller must ensure that the requested space is not yet available, and that the requested + * space is less than BUFFER_SIZE. + * + * @throws InvalidProtocolBufferException The end of the stream or the current limit was + * reached. + */ + private void refillBuffer(int n) throws IOException { + if (!tryRefillBuffer(n)) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + } + + /** + * Tries to read more bytes from the input, making at least {@code n} bytes available in the + * buffer. Caller must ensure that the requested space is not yet available, and that the + * requested space is less than BUFFER_SIZE. + * + * @return {@code true} if the bytes could be made available; {@code false} if the end of the + * stream or the current limit was reached. + */ + private boolean tryRefillBuffer(int n) throws IOException { + if (pos + n <= bufferSize) { + throw new IllegalStateException( + "refillBuffer() called when " + n + " bytes were already available in buffer"); + } + + if (totalBytesRetired + pos + n > currentLimit) { + // Oops, we hit a limit. + return false; + } + + if (refillCallback != null) { + refillCallback.onRefill(); + } + + int tempPos = pos; + if (tempPos > 0) { + if (bufferSize > tempPos) { + System.arraycopy(buffer, tempPos, buffer, 0, bufferSize - tempPos); + } + totalBytesRetired += tempPos; + bufferSize -= tempPos; + pos = 0; + } + + int bytesRead = input.read(buffer, bufferSize, buffer.length - bufferSize); + if (bytesRead == 0 || bytesRead < -1 || bytesRead > buffer.length) { + throw new IllegalStateException( + "InputStream#read(byte[]) returned invalid result: " + + bytesRead + + "\nThe InputStream implementation is buggy."); + } + if (bytesRead > 0) { + bufferSize += bytesRead; + // Integer-overflow-conscious check against sizeLimit + if (totalBytesRetired + n - sizeLimit > 0) { + throw InvalidProtocolBufferException.sizeLimitExceeded(); + } + recomputeBufferSizeAfterLimit(); + return (bufferSize >= n) ? true : tryRefillBuffer(n); + } + + return false; + } + + @Override + public byte readRawByte() throws IOException { + if (pos == bufferSize) { + refillBuffer(1); + } + return buffer[pos++]; + } + + @Override + public byte[] readRawBytes(final int size) throws IOException { + final int tempPos = pos; + if (size <= (bufferSize - tempPos) && size > 0) { + pos = tempPos + size; + return Arrays.copyOfRange(buffer, tempPos, tempPos + size); + } else { + return readRawBytesSlowPath(size); + } + } + + /** + * Exactly like readRawBytes, but caller must have already checked the fast path: (size <= + * (bufferSize - pos) && size > 0) + */ + private byte[] readRawBytesSlowPath(final int size) throws IOException { + if (size == 0) { + return Internal.EMPTY_BYTE_ARRAY; + } + if (size < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + + // Verify that the message size so far has not exceeded sizeLimit. + int currentMessageSize = totalBytesRetired + pos + size; + if (currentMessageSize > sizeLimit) { + throw InvalidProtocolBufferException.sizeLimitExceeded(); + } + + // Verify that the message size so far has not exceeded currentLimit. + if (currentMessageSize > currentLimit) { + // Read to the end of the stream anyway. + skipRawBytes(currentLimit - totalBytesRetired - pos); + throw InvalidProtocolBufferException.truncatedMessage(); + } + + final int originalBufferPos = pos; + final int bufferedBytes = bufferSize - pos; + + // Mark the current buffer consumed. + totalBytesRetired += bufferSize; + pos = 0; + bufferSize = 0; + + // Determine the number of bytes we need to read from the input stream. + int sizeLeft = size - bufferedBytes; + // TODO(nathanmittler): Consider using a value larger than DEFAULT_BUFFER_SIZE. + if (sizeLeft < DEFAULT_BUFFER_SIZE || sizeLeft <= input.available()) { + // Either the bytes we need are known to be available, or the required buffer is + // within an allowed threshold - go ahead and allocate the buffer now. + final byte[] bytes = new byte[size]; + + // Copy all of the buffered bytes to the result buffer. + System.arraycopy(buffer, originalBufferPos, bytes, 0, bufferedBytes); + + // Fill the remaining bytes from the input stream. + int tempPos = bufferedBytes; + while (tempPos < bytes.length) { + int n = input.read(bytes, tempPos, size - tempPos); + if (n == -1) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + totalBytesRetired += n; + tempPos += n; + } + + return bytes; + } + + // The size is very large. For security reasons, we can't allocate the + // entire byte array yet. The size comes directly from the input, so a + // maliciously-crafted message could provide a bogus very large size in + // order to trick the app into allocating a lot of memory. We avoid this + // by allocating and reading only a small chunk at a time, so that the + // malicious message must actually *be* extremely large to cause + // problems. Meanwhile, we limit the allowed size of a message elsewhere. + final List<byte[]> chunks = new ArrayList<byte[]>(); + + while (sizeLeft > 0) { + // TODO(nathanmittler): Consider using a value larger than DEFAULT_BUFFER_SIZE. + final byte[] chunk = new byte[Math.min(sizeLeft, DEFAULT_BUFFER_SIZE)]; + int tempPos = 0; + while (tempPos < chunk.length) { + final int n = input.read(chunk, tempPos, chunk.length - tempPos); + if (n == -1) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + totalBytesRetired += n; + tempPos += n; + } + sizeLeft -= chunk.length; + chunks.add(chunk); + } + + // OK, got everything. Now concatenate it all into one buffer. + final byte[] bytes = new byte[size]; + + // Start by copying the leftover bytes from this.buffer. + System.arraycopy(buffer, originalBufferPos, bytes, 0, bufferedBytes); + + // And now all the chunks. + int tempPos = bufferedBytes; + for (final byte[] chunk : chunks) { + System.arraycopy(chunk, 0, bytes, tempPos, chunk.length); + tempPos += chunk.length; + } + + // Done. + return bytes; + } + + @Override + public void skipRawBytes(final int size) throws IOException { + if (size <= (bufferSize - pos) && size >= 0) { + // We have all the bytes we need already. + pos += size; + } else { + skipRawBytesSlowPath(size); + } + } + + /** + * Exactly like skipRawBytes, but caller must have already checked the fast path: (size <= + * (bufferSize - pos) && size >= 0) + */ + private void skipRawBytesSlowPath(final int size) throws IOException { + if (size < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + + if (totalBytesRetired + pos + size > currentLimit) { + // Read to the end of the stream anyway. + skipRawBytes(currentLimit - totalBytesRetired - pos); + // Then fail. + throw InvalidProtocolBufferException.truncatedMessage(); + } + + // Skipping more bytes than are in the buffer. First skip what we have. + int tempPos = bufferSize - pos; + pos = bufferSize; + + // Keep refilling the buffer until we get to the point we wanted to skip to. + // This has the side effect of ensuring the limits are updated correctly. + refillBuffer(1); + while (size - tempPos > bufferSize) { + tempPos += bufferSize; + pos = bufferSize; + refillBuffer(1); + } + + pos = size - tempPos; + } } } diff --git a/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java b/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java index e5515285..3e32c2c5 100644 --- a/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java +++ b/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java @@ -30,10 +30,12 @@ package com.google.protobuf; +import static com.google.protobuf.WireFormat.FIXED_32_SIZE; +import static com.google.protobuf.WireFormat.FIXED_64_SIZE; +import static com.google.protobuf.WireFormat.MAX_VARINT_SIZE; import static java.lang.Math.max; import com.google.protobuf.Utf8.UnpairedSurrogateException; - import java.io.IOException; import java.io.OutputStream; import java.nio.BufferOverflowException; @@ -59,10 +61,6 @@ public abstract class CodedOutputStream extends ByteOutput { private static final boolean HAS_UNSAFE_ARRAY_OPERATIONS = UnsafeUtil.hasUnsafeArrayOperations(); private static final long ARRAY_BASE_OFFSET = UnsafeUtil.getArrayBaseOffset(); - private static final int FIXED_32_SIZE = 4; - private static final int FIXED_64_SIZE = 8; - private static final int MAX_VARINT_SIZE = 10; - /** * @deprecated Use {@link #computeFixed32SizeNoTag(int)} instead. */ @@ -134,14 +132,27 @@ public abstract class CodedOutputStream extends ByteOutput { return new ArrayEncoder(flatArray, offset, length); } - /** - * Create a new {@code CodedOutputStream} that writes to the given {@link ByteBuffer}. - */ - public static CodedOutputStream newInstance(ByteBuffer byteBuffer) { - if (byteBuffer.hasArray()) { - return new NioHeapEncoder(byteBuffer); + /** Create a new {@code CodedOutputStream} that writes to the given {@link ByteBuffer}. */ + public static CodedOutputStream newInstance(ByteBuffer buffer) { + if (buffer.hasArray()) { + return new HeapNioEncoder(buffer); } - return new NioEncoder(byteBuffer); + if (buffer.isDirect() && !buffer.isReadOnly()) { + return UnsafeDirectNioEncoder.isSupported() + ? newUnsafeInstance(buffer) + : newSafeInstance(buffer); + } + throw new IllegalArgumentException("ByteBuffer is read-only"); + } + + /** For testing purposes only. */ + static CodedOutputStream newUnsafeInstance(ByteBuffer buffer) { + return new UnsafeDirectNioEncoder(buffer); + } + + /** For testing purposes only. */ + static CodedOutputStream newSafeInstance(ByteBuffer buffer) { + return new SafeDirectNioEncoder(buffer); } /** @@ -979,6 +990,10 @@ public abstract class CodedOutputStream extends ByteOutput { super(MESSAGE); } + OutOfSpaceException(String explanationMessage) { + super(MESSAGE + ": " + explanationMessage); + } + OutOfSpaceException(Throwable cause) { super(MESSAGE, cause); } @@ -1486,11 +1501,11 @@ public abstract class CodedOutputStream extends ByteOutput { * A {@link CodedOutputStream} that writes directly to a heap {@link ByteBuffer}. Writes are * done directly to the underlying array. The buffer position is only updated after a flush. */ - private static final class NioHeapEncoder extends ArrayEncoder { + private static final class HeapNioEncoder extends ArrayEncoder { private final ByteBuffer byteBuffer; private int initialPosition; - NioHeapEncoder(ByteBuffer byteBuffer) { + HeapNioEncoder(ByteBuffer byteBuffer) { super(byteBuffer.array(), byteBuffer.arrayOffset() + byteBuffer.position(), byteBuffer.remaining()); this.byteBuffer = byteBuffer; @@ -1505,14 +1520,15 @@ public abstract class CodedOutputStream extends ByteOutput { } /** - * A {@link CodedOutputStream} that writes directly to a {@link ByteBuffer}. + * A {@link CodedOutputStream} that writes directly to a direct {@link ByteBuffer}, using only + * safe operations.. */ - private static final class NioEncoder extends CodedOutputStream { + private static final class SafeDirectNioEncoder extends CodedOutputStream { private final ByteBuffer originalBuffer; private final ByteBuffer buffer; private final int initialPosition; - NioEncoder(ByteBuffer buffer) { + SafeDirectNioEncoder(ByteBuffer buffer) { this.originalBuffer = buffer; this.buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); initialPosition = buffer.position(); @@ -1815,6 +1831,356 @@ public abstract class CodedOutputStream extends ByteOutput { } /** + * A {@link CodedOutputStream} that writes directly to a direct {@link ByteBuffer} using {@code + * sun.misc.Unsafe}. + */ + private static final class UnsafeDirectNioEncoder extends CodedOutputStream { + private final ByteBuffer originalBuffer; + private final ByteBuffer buffer; + private final long address; + private final long initialPosition; + private final long limit; + private final long oneVarintLimit; + private long position; + + UnsafeDirectNioEncoder(ByteBuffer buffer) { + this.originalBuffer = buffer; + this.buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + address = UnsafeUtil.addressOffset(buffer); + initialPosition = address + buffer.position(); + limit = address + buffer.limit(); + oneVarintLimit = limit - MAX_VARINT_SIZE; + position = initialPosition; + } + + static boolean isSupported() { + return UnsafeUtil.hasUnsafeByteBufferOperations(); + } + + @Override + public void writeTag(int fieldNumber, int wireType) throws IOException { + writeUInt32NoTag(WireFormat.makeTag(fieldNumber, wireType)); + } + + @Override + public void writeInt32(int fieldNumber, int value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + writeInt32NoTag(value); + } + + @Override + public void writeUInt32(int fieldNumber, int value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + writeUInt32NoTag(value); + } + + @Override + public void writeFixed32(int fieldNumber, int value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED32); + writeFixed32NoTag(value); + } + + @Override + public void writeUInt64(int fieldNumber, long value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + writeUInt64NoTag(value); + } + + @Override + public void writeFixed64(int fieldNumber, long value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED64); + writeFixed64NoTag(value); + } + + @Override + public void writeBool(int fieldNumber, boolean value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + write((byte) (value ? 1 : 0)); + } + + @Override + public void writeString(int fieldNumber, String value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeStringNoTag(value); + } + + @Override + public void writeBytes(int fieldNumber, ByteString value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeBytesNoTag(value); + } + + @Override + public void writeByteArray(int fieldNumber, byte[] value) throws IOException { + writeByteArray(fieldNumber, value, 0, value.length); + } + + @Override + public void writeByteArray(int fieldNumber, byte[] value, int offset, int length) + throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeByteArrayNoTag(value, offset, length); + } + + @Override + public void writeByteBuffer(int fieldNumber, ByteBuffer value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeUInt32NoTag(value.capacity()); + writeRawBytes(value); + } + + @Override + public void writeMessage(int fieldNumber, MessageLite value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeMessageNoTag(value); + } + + @Override + public void writeMessageSetExtension(int fieldNumber, MessageLite value) throws IOException { + writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP); + writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber); + writeMessage(WireFormat.MESSAGE_SET_MESSAGE, value); + writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_END_GROUP); + } + + @Override + public void writeRawMessageSetExtension(int fieldNumber, ByteString value) throws IOException { + writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP); + writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber); + writeBytes(WireFormat.MESSAGE_SET_MESSAGE, value); + writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_END_GROUP); + } + + @Override + public void writeMessageNoTag(MessageLite value) throws IOException { + writeUInt32NoTag(value.getSerializedSize()); + value.writeTo(this); + } + + @Override + public void write(byte value) throws IOException { + if (position >= limit) { + throw new OutOfSpaceException( + String.format("Pos: %d, limit: %d, len: %d", position, limit, 1)); + } + UnsafeUtil.putByte(position++, value); + } + + @Override + public void writeBytesNoTag(ByteString value) throws IOException { + writeUInt32NoTag(value.size()); + value.writeTo(this); + } + + @Override + public void writeByteArrayNoTag(byte[] value, int offset, int length) throws IOException { + writeUInt32NoTag(length); + write(value, offset, length); + } + + @Override + public void writeRawBytes(ByteBuffer value) throws IOException { + if (value.hasArray()) { + write(value.array(), value.arrayOffset(), value.capacity()); + } else { + ByteBuffer duplicated = value.duplicate(); + duplicated.clear(); + write(duplicated); + } + } + + @Override + public void writeInt32NoTag(int value) throws IOException { + if (value >= 0) { + writeUInt32NoTag(value); + } else { + // Must sign-extend. + writeUInt64NoTag(value); + } + } + + @Override + public void writeUInt32NoTag(int value) throws IOException { + if (position <= oneVarintLimit) { + // Optimization to avoid bounds checks on each iteration. + while (true) { + if ((value & ~0x7F) == 0) { + UnsafeUtil.putByte(position++, (byte) value); + return; + } else { + UnsafeUtil.putByte(position++, (byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + } + } else { + while (position < limit) { + if ((value & ~0x7F) == 0) { + UnsafeUtil.putByte(position++, (byte) value); + return; + } else { + UnsafeUtil.putByte(position++, (byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + } + throw new OutOfSpaceException( + String.format("Pos: %d, limit: %d, len: %d", position, limit, 1)); + } + } + + @Override + public void writeFixed32NoTag(int value) throws IOException { + buffer.putInt(bufferPos(position), value); + position += FIXED_32_SIZE; + } + + @Override + public void writeUInt64NoTag(long value) throws IOException { + if (position <= oneVarintLimit) { + // Optimization to avoid bounds checks on each iteration. + while (true) { + if ((value & ~0x7FL) == 0) { + UnsafeUtil.putByte(position++, (byte) value); + return; + } else { + UnsafeUtil.putByte(position++, (byte) (((int) value & 0x7F) | 0x80)); + value >>>= 7; + } + } + } else { + while (position < limit) { + if ((value & ~0x7FL) == 0) { + UnsafeUtil.putByte(position++, (byte) value); + return; + } else { + UnsafeUtil.putByte(position++, (byte) (((int) value & 0x7F) | 0x80)); + value >>>= 7; + } + } + throw new OutOfSpaceException( + String.format("Pos: %d, limit: %d, len: %d", position, limit, 1)); + } + } + + @Override + public void writeFixed64NoTag(long value) throws IOException { + buffer.putLong(bufferPos(position), value); + position += FIXED_64_SIZE; + } + + @Override + public void write(byte[] value, int offset, int length) throws IOException { + if (value == null + || offset < 0 + || length < 0 + || (value.length - length) < offset + || (limit - length) < position) { + if (value == null) { + throw new NullPointerException("value"); + } + throw new OutOfSpaceException( + String.format("Pos: %d, limit: %d, len: %d", position, limit, length)); + } + + UnsafeUtil.copyMemory( + value, UnsafeUtil.getArrayBaseOffset() + offset, null, position, length); + position += length; + } + + @Override + public void writeLazy(byte[] value, int offset, int length) throws IOException { + write(value, offset, length); + } + + @Override + public void write(ByteBuffer value) throws IOException { + try { + int length = value.remaining(); + repositionBuffer(position); + buffer.put(value); + position += length; + } catch (BufferOverflowException e) { + throw new OutOfSpaceException(e); + } + } + + @Override + public void writeLazy(ByteBuffer value) throws IOException { + write(value); + } + + @Override + public void writeStringNoTag(String value) throws IOException { + long prevPos = position; + try { + // UTF-8 byte length of the string is at least its UTF-16 code unit length (value.length()), + // and at most 3 times of it. We take advantage of this in both branches below. + int maxEncodedSize = value.length() * Utf8.MAX_BYTES_PER_CHAR; + int maxLengthVarIntSize = computeUInt32SizeNoTag(maxEncodedSize); + int minLengthVarIntSize = computeUInt32SizeNoTag(value.length()); + if (minLengthVarIntSize == maxLengthVarIntSize) { + // Save the current position and increment past the length field. We'll come back + // and write the length field after the encoding is complete. + int stringStart = bufferPos(position) + minLengthVarIntSize; + buffer.position(stringStart); + + // Encode the string. + Utf8.encodeUtf8(value, buffer); + + // Write the length and advance the position. + int length = buffer.position() - stringStart; + writeUInt32NoTag(length); + position += length; + } else { + // Calculate and write the encoded length. + int length = Utf8.encodedLength(value); + writeUInt32NoTag(length); + + // Write the string and advance the position. + repositionBuffer(position); + Utf8.encodeUtf8(value, buffer); + position += length; + } + } catch (UnpairedSurrogateException e) { + // Roll back the change and convert to an IOException. + position = prevPos; + repositionBuffer(position); + + // TODO(nathanmittler): We should throw an IOException here instead. + inefficientWriteStringNoTag(value, e); + } catch (IllegalArgumentException e) { + // Thrown by buffer.position() if out of range. + throw new OutOfSpaceException(e); + } catch (IndexOutOfBoundsException e) { + throw new OutOfSpaceException(e); + } + } + + @Override + public void flush() { + // Update the position of the original buffer. + originalBuffer.position(bufferPos(position)); + } + + @Override + public int spaceLeft() { + return (int) (limit - position); + } + + @Override + public int getTotalBytesWritten() { + return (int) (position - initialPosition); + } + + private void repositionBuffer(long pos) { + buffer.position(bufferPos(pos)); + } + + private int bufferPos(long pos) { + return (int) (pos - address); + } + } + + /** * Abstract base class for buffered encoders. */ private abstract static class AbstractBufferedEncoder extends CodedOutputStream { diff --git a/java/core/src/main/java/com/google/protobuf/Descriptors.java b/java/core/src/main/java/com/google/protobuf/Descriptors.java index 1c34c24f..cab7118a 100644 --- a/java/core/src/main/java/com/google/protobuf/Descriptors.java +++ b/java/core/src/main/java/com/google/protobuf/Descriptors.java @@ -1212,33 +1212,20 @@ public final class Descriptors { private final Object defaultDefault; } - // TODO(xiaofeng): Implement it consistently across different languages. See b/24751348. - private static String fieldNameToLowerCamelCase(String name) { + // This method should match exactly with the ToJsonName() function in C++ + // descriptor.cc. + private static String fieldNameToJsonName(String name) { StringBuilder result = new StringBuilder(name.length()); boolean isNextUpperCase = false; for (int i = 0; i < name.length(); i++) { Character ch = name.charAt(i); - if (Character.isLowerCase(ch)) { - if (isNextUpperCase) { - result.append(Character.toUpperCase(ch)); - } else { - result.append(ch); - } - isNextUpperCase = false; - } else if (Character.isUpperCase(ch)) { - if (i == 0) { - // Force first letter to lower-case. - result.append(Character.toLowerCase(ch)); - } else { - // Capital letters after the first are left as-is. - result.append(ch); - } - isNextUpperCase = false; - } else if (Character.isDigit(ch)) { - result.append(ch); + if (ch == '_') { + isNextUpperCase = true; + } else if (isNextUpperCase) { + result.append(Character.toUpperCase(ch)); isNextUpperCase = false; } else { - isNextUpperCase = true; + result.append(ch); } } return result.toString(); @@ -1257,7 +1244,7 @@ public final class Descriptors { if (proto.hasJsonName()) { jsonName = proto.getJsonName(); } else { - jsonName = fieldNameToLowerCamelCase(proto.getName()); + jsonName = fieldNameToJsonName(proto.getName()); } if (proto.hasType()) { diff --git a/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java b/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java index 5dfe7ff7..5c0b9967 100644 --- a/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java +++ b/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java @@ -36,7 +36,6 @@ import com.google.protobuf.Descriptors.EnumValueDescriptor; import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.Descriptors.FileDescriptor; import com.google.protobuf.Descriptors.OneofDescriptor; -import com.google.protobuf.GeneratedMessage.GeneratedExtension; import java.io.IOException; import java.io.InputStream; @@ -45,6 +44,7 @@ import java.io.Serializable; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -1609,6 +1609,43 @@ public abstract class GeneratedMessageV3 extends AbstractMessage FieldDescriptor getDescriptor(); } + /** For use by generated code only. */ + public static <ContainingType extends Message, Type> + GeneratedExtension<ContainingType, Type> + newMessageScopedGeneratedExtension(final Message scope, + final int descriptorIndex, + final Class singularType, + final Message defaultInstance) { + // For extensions scoped within a Message, we use the Message to resolve + // the outer class's descriptor, from which the extension descriptor is + // obtained. + return new GeneratedExtension<ContainingType, Type>( + new CachedDescriptorRetriever() { + @Override + public FieldDescriptor loadDescriptor() { + return scope.getDescriptorForType().getExtensions().get(descriptorIndex); + } + }, + singularType, + defaultInstance, + Extension.ExtensionType.IMMUTABLE); + } + + /** For use by generated code only. */ + public static <ContainingType extends Message, Type> + GeneratedExtension<ContainingType, Type> + newFileScopedGeneratedExtension(final Class singularType, + final Message defaultInstance) { + // For extensions scoped within a file, we rely on the outer class's + // static initializer to call internalInit() on the extension when the + // descriptor is available. + return new GeneratedExtension<ContainingType, Type>( + null, // ExtensionDescriptorRetriever is initialized in internalInit(); + singularType, + defaultInstance, + Extension.ExtensionType.IMMUTABLE); + } + private abstract static class CachedDescriptorRetriever implements ExtensionDescriptorRetriever { private volatile FieldDescriptor descriptor; @@ -1627,6 +1664,301 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } } + /** + * Used in proto1 generated code only. + * + * After enabling bridge, we can define proto2 extensions (the extended type + * is a proto2 mutable message) in a proto1 .proto file. For these extensions + * we should generate proto2 GeneratedExtensions. + */ + public static <ContainingType extends Message, Type> + GeneratedExtension<ContainingType, Type> + newMessageScopedGeneratedExtension( + final Message scope, final String name, + final Class singularType, final Message defaultInstance) { + // For extensions scoped within a Message, we use the Message to resolve + // the outer class's descriptor, from which the extension descriptor is + // obtained. + return new GeneratedExtension<ContainingType, Type>( + new CachedDescriptorRetriever() { + @Override + protected FieldDescriptor loadDescriptor() { + return scope.getDescriptorForType().findFieldByName(name); + } + }, + singularType, + defaultInstance, + Extension.ExtensionType.MUTABLE); + } + + /** + * Used in proto1 generated code only. + * + * After enabling bridge, we can define proto2 extensions (the extended type + * is a proto2 mutable message) in a proto1 .proto file. For these extensions + * we should generate proto2 GeneratedExtensions. + */ + public static <ContainingType extends Message, Type> + GeneratedExtension<ContainingType, Type> + newFileScopedGeneratedExtension( + final Class singularType, final Message defaultInstance, + final String descriptorOuterClass, final String extensionName) { + // For extensions scoped within a file, we load the descriptor outer + // class and rely on it to get the FileDescriptor which then can be + // used to obtain the extension's FieldDescriptor. + return new GeneratedExtension<ContainingType, Type>( + new CachedDescriptorRetriever() { + @Override + protected FieldDescriptor loadDescriptor() { + try { + Class clazz = singularType.getClassLoader().loadClass(descriptorOuterClass); + FileDescriptor file = (FileDescriptor) clazz.getField("descriptor").get(null); + return file.findExtensionByName(extensionName); + } catch (Exception e) { + throw new RuntimeException( + "Cannot load descriptors: " + + descriptorOuterClass + + " is not a valid descriptor class name", + e); + } + } + }, + singularType, + defaultInstance, + Extension.ExtensionType.MUTABLE); + } + + /** + * Type used to represent generated extensions. The protocol compiler + * generates a static singleton instance of this class for each extension. + * + * <p>For example, imagine you have the {@code .proto} file: + * + * <pre> + * option java_class = "MyProto"; + * + * message Foo { + * extensions 1000 to max; + * } + * + * extend Foo { + * optional int32 bar; + * } + * </pre> + * + * <p>Then, {@code MyProto.Foo.bar} has type + * {@code GeneratedExtension<MyProto.Foo, Integer>}. + * + * <p>In general, users should ignore the details of this type, and simply use + * these static singletons as parameters to the extension accessors defined + * in {@link ExtendableMessage} and {@link ExtendableBuilder}. + */ + public static class GeneratedExtension< + ContainingType extends Message, Type> extends + Extension<ContainingType, Type> { + // TODO(kenton): Find ways to avoid using Java reflection within this + // class. Also try to avoid suppressing unchecked warnings. + + // We can't always initialize the descriptor of a GeneratedExtension when + // we first construct it due to initialization order difficulties (namely, + // the descriptor may not have been constructed yet, since it is often + // constructed by the initializer of a separate module). + // + // In the case of nested extensions, we initialize the + // ExtensionDescriptorRetriever with an instance that uses the scoping + // Message's default instance to retrieve the extension's descriptor. + // + // In the case of non-nested extensions, we initialize the + // ExtensionDescriptorRetriever to null and rely on the outer class's static + // initializer to call internalInit() after the descriptor has been parsed. + GeneratedExtension(ExtensionDescriptorRetriever descriptorRetriever, + Class singularType, + Message messageDefaultInstance, + ExtensionType extensionType) { + if (Message.class.isAssignableFrom(singularType) && + !singularType.isInstance(messageDefaultInstance)) { + throw new IllegalArgumentException( + "Bad messageDefaultInstance for " + singularType.getName()); + } + this.descriptorRetriever = descriptorRetriever; + this.singularType = singularType; + this.messageDefaultInstance = messageDefaultInstance; + + if (ProtocolMessageEnum.class.isAssignableFrom(singularType)) { + this.enumValueOf = getMethodOrDie(singularType, "valueOf", + EnumValueDescriptor.class); + this.enumGetValueDescriptor = + getMethodOrDie(singularType, "getValueDescriptor"); + } else { + this.enumValueOf = null; + this.enumGetValueDescriptor = null; + } + this.extensionType = extensionType; + } + + /** For use by generated code only. */ + public void internalInit(final FieldDescriptor descriptor) { + if (descriptorRetriever != null) { + throw new IllegalStateException("Already initialized."); + } + descriptorRetriever = + new ExtensionDescriptorRetriever() { + @Override + public FieldDescriptor getDescriptor() { + return descriptor; + } + }; + } + + private ExtensionDescriptorRetriever descriptorRetriever; + private final Class singularType; + private final Message messageDefaultInstance; + private final Method enumValueOf; + private final Method enumGetValueDescriptor; + private final ExtensionType extensionType; + + @Override + public FieldDescriptor getDescriptor() { + if (descriptorRetriever == null) { + throw new IllegalStateException( + "getDescriptor() called before internalInit()"); + } + return descriptorRetriever.getDescriptor(); + } + + /** + * If the extension is an embedded message or group, returns the default + * instance of the message. + */ + @Override + public Message getMessageDefaultInstance() { + return messageDefaultInstance; + } + + @Override + protected ExtensionType getExtensionType() { + return extensionType; + } + + /** + * Convert from the type used by the reflection accessors to the type used + * by native accessors. E.g., for enums, the reflection accessors use + * EnumValueDescriptors but the native accessors use the generated enum + * type. + */ + @Override + @SuppressWarnings("unchecked") + protected Object fromReflectionType(final Object value) { + FieldDescriptor descriptor = getDescriptor(); + if (descriptor.isRepeated()) { + if (descriptor.getJavaType() == FieldDescriptor.JavaType.MESSAGE || + descriptor.getJavaType() == FieldDescriptor.JavaType.ENUM) { + // Must convert the whole list. + final List result = new ArrayList(); + for (final Object element : (List) value) { + result.add(singularFromReflectionType(element)); + } + return result; + } else { + return value; + } + } else { + return singularFromReflectionType(value); + } + } + + /** + * Like {@link #fromReflectionType(Object)}, but if the type is a repeated + * type, this converts a single element. + */ + @Override + protected Object singularFromReflectionType(final Object value) { + FieldDescriptor descriptor = getDescriptor(); + switch (descriptor.getJavaType()) { + case MESSAGE: + if (singularType.isInstance(value)) { + return value; + } else { + return messageDefaultInstance.newBuilderForType() + .mergeFrom((Message) value).build(); + } + case ENUM: + return invokeOrDie(enumValueOf, null, (EnumValueDescriptor) value); + default: + return value; + } + } + + /** + * Convert from the type used by the native accessors to the type used + * by reflection accessors. E.g., for enums, the reflection accessors use + * EnumValueDescriptors but the native accessors use the generated enum + * type. + */ + @Override + @SuppressWarnings("unchecked") + protected Object toReflectionType(final Object value) { + FieldDescriptor descriptor = getDescriptor(); + if (descriptor.isRepeated()) { + if (descriptor.getJavaType() == FieldDescriptor.JavaType.ENUM) { + // Must convert the whole list. + final List result = new ArrayList(); + for (final Object element : (List) value) { + result.add(singularToReflectionType(element)); + } + return result; + } else { + return value; + } + } else { + return singularToReflectionType(value); + } + } + + /** + * Like {@link #toReflectionType(Object)}, but if the type is a repeated + * type, this converts a single element. + */ + @Override + protected Object singularToReflectionType(final Object value) { + FieldDescriptor descriptor = getDescriptor(); + switch (descriptor.getJavaType()) { + case ENUM: + return invokeOrDie(enumGetValueDescriptor, value); + default: + return value; + } + } + + @Override + public int getNumber() { + return getDescriptor().getNumber(); + } + + @Override + public WireFormat.FieldType getLiteType() { + return getDescriptor().getLiteType(); + } + + @Override + public boolean isRepeated() { + return getDescriptor().isRepeated(); + } + + @Override + @SuppressWarnings("unchecked") + public Type getDefaultValue() { + if (isRepeated()) { + return (Type) Collections.emptyList(); + } + if (getDescriptor().getJavaType() == FieldDescriptor.JavaType.MESSAGE) { + return (Type) messageDefaultInstance; + } + return (Type) singularFromReflectionType( + getDescriptor().getDefaultValue()); + } + } + // ================================================================= /** Calls Class.getMethod and throws a RuntimeException if it fails. */ @@ -2223,6 +2555,20 @@ public abstract class GeneratedMessageV3 extends AbstractMessage field.getNumber()); } + private Message coerceType(Message value) { + if (value == null) { + return null; + } + if (mapEntryMessageDefaultInstance.getClass().isInstance(value)) { + return value; + } + // The value is not the exact right message type. However, if it + // is an alternative implementation of the same type -- e.g. a + // DynamicMessage -- we should accept it. In this case we can make + // a copy of the message. + return mapEntryMessageDefaultInstance.toBuilder().mergeFrom(value).build(); + } + @Override public Object get(GeneratedMessageV3 message) { List result = new ArrayList(); @@ -2278,15 +2624,15 @@ public abstract class GeneratedMessageV3 extends AbstractMessage public Object getRepeatedRaw(Builder builder, int index) { return getRepeated(builder, index); } - + @Override public void setRepeated(Builder builder, int index, Object value) { - getMutableMapField(builder).getMutableList().set(index, (Message) value); + getMutableMapField(builder).getMutableList().set(index, coerceType((Message) value)); } @Override public void addRepeated(Builder builder, Object value) { - getMutableMapField(builder).getMutableList().add((Message) value); + getMutableMapField(builder).getMutableList().add(coerceType((Message) value)); } @Override @@ -2713,4 +3059,131 @@ public abstract class GeneratedMessageV3 extends AbstractMessage output.writeBytesNoTag((ByteString) value); } } + + protected static <V> void serializeIntegerMapTo( + CodedOutputStream out, + MapField<Integer, V> field, + MapEntry<Integer, V> defaultEntry, + int fieldNumber) throws IOException { + Map<Integer, V> m = field.getMap(); + if (!out.isSerializationDeterministic()) { + serializeMapTo(out, m, defaultEntry, fieldNumber); + return; + } + // Sorting the unboxed keys and then look up the values during serialziation is 2x faster + // than sorting map entries with a custom comparator directly. + int[] keys = new int[m.size()]; + int index = 0; + for (int k : m.keySet()) { + keys[index++] = k; + } + Arrays.sort(keys); + for (int key : keys) { + out.writeMessage(fieldNumber, + defaultEntry.newBuilderForType() + .setKey(key) + .setValue(m.get(key)) + .build()); + } + } + + protected static <V> void serializeLongMapTo( + CodedOutputStream out, + MapField<Long, V> field, + MapEntry<Long, V> defaultEntry, + int fieldNumber) + throws IOException { + Map<Long, V> m = field.getMap(); + if (!out.isSerializationDeterministic()) { + serializeMapTo(out, m, defaultEntry, fieldNumber); + return; + } + + long[] keys = new long[m.size()]; + int index = 0; + for (long k : m.keySet()) { + keys[index++] = k; + } + Arrays.sort(keys); + for (long key : keys) { + out.writeMessage(fieldNumber, + defaultEntry.newBuilderForType() + .setKey(key) + .setValue(m.get(key)) + .build()); + } + } + + protected static <V> void serializeStringMapTo( + CodedOutputStream out, + MapField<String, V> field, + MapEntry<String, V> defaultEntry, + int fieldNumber) + throws IOException { + Map<String, V> m = field.getMap(); + if (!out.isSerializationDeterministic()) { + serializeMapTo(out, m, defaultEntry, fieldNumber); + return; + } + + // Sorting the String keys and then look up the values during serialziation is 25% faster than + // sorting map entries with a custom comparator directly. + String[] keys = new String[m.size()]; + keys = m.keySet().toArray(keys); + Arrays.sort(keys); + for (String key : keys) { + out.writeMessage(fieldNumber, + defaultEntry.newBuilderForType() + .setKey(key) + .setValue(m.get(key)) + .build()); + } + } + + protected static <V> void serializeBooleanMapTo( + CodedOutputStream out, + MapField<Boolean, V> field, + MapEntry<Boolean, V> defaultEntry, + int fieldNumber) + throws IOException { + Map<Boolean, V> m = field.getMap(); + if (!out.isSerializationDeterministic()) { + serializeMapTo(out, m, defaultEntry, fieldNumber); + return; + } + maybeSerializeBooleanEntryTo(out, m, defaultEntry, fieldNumber, false); + maybeSerializeBooleanEntryTo(out, m, defaultEntry, fieldNumber, true); + } + + private static <V> void maybeSerializeBooleanEntryTo( + CodedOutputStream out, + Map<Boolean, V> m, + MapEntry<Boolean, V> defaultEntry, + int fieldNumber, + boolean key) + throws IOException { + if (m.containsKey(key)) { + out.writeMessage(fieldNumber, + defaultEntry.newBuilderForType() + .setKey(key) + .setValue(m.get(key)) + .build()); + } + } + + /** Serialize the map using the iteration order. */ + private static <K, V> void serializeMapTo( + CodedOutputStream out, + Map<K, V> m, + MapEntry<K, V> defaultEntry, + int fieldNumber) + throws IOException { + for (Map.Entry<K, V> entry : m.entrySet()) { + out.writeMessage(fieldNumber, + defaultEntry.newBuilderForType() + .setKey(entry.getKey()) + .setValue(entry.getValue()) + .build()); + } + } } diff --git a/java/core/src/main/java/com/google/protobuf/Internal.java b/java/core/src/main/java/com/google/protobuf/Internal.java index 3b4a0412..c234559c 100644 --- a/java/core/src/main/java/com/google/protobuf/Internal.java +++ b/java/core/src/main/java/com/google/protobuf/Internal.java @@ -60,6 +60,16 @@ public final class Internal { static final Charset ISO_8859_1 = Charset.forName("ISO-8859-1"); /** + * Throws an appropriate {@link NullPointerException} if the given objects is {@code null}. + */ + static <T> T checkNotNull(T obj, String message) { + if (obj == null) { + throw new NullPointerException(message); + } + return obj; + } + + /** * Helper called by generated code to construct default values for string * fields. * <p> diff --git a/java/core/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java b/java/core/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java index 85ce7b24..55e33d21 100644 --- a/java/core/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java +++ b/java/core/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java @@ -107,11 +107,23 @@ public class InvalidProtocolBufferException extends IOException { "Protocol message end-group tag did not match expected tag."); } - static InvalidProtocolBufferException invalidWireType() { - return new InvalidProtocolBufferException( + static InvalidWireTypeException invalidWireType() { + return new InvalidWireTypeException( "Protocol message tag had invalid wire type."); } + /** + * Exception indicating that and unexpected wire type was encountered for a field. + */ + @ExperimentalApi + public static class InvalidWireTypeException extends InvalidProtocolBufferException { + private static final long serialVersionUID = 3283890091615336259L; + + public InvalidWireTypeException(String description) { + super(description); + } + } + static InvalidProtocolBufferException recursionLimitExceeded() { return new InvalidProtocolBufferException( "Protocol message had too many levels of nesting. May be malicious. " + diff --git a/java/core/src/main/java/com/google/protobuf/MapEntry.java b/java/core/src/main/java/com/google/protobuf/MapEntry.java index 117cd911..4f0351f4 100644 --- a/java/core/src/main/java/com/google/protobuf/MapEntry.java +++ b/java/core/src/main/java/com/google/protobuf/MapEntry.java @@ -334,6 +334,15 @@ public final class MapEntry<K, V> extends AbstractMessage { } else { if (field.getType() == FieldDescriptor.Type.ENUM) { value = ((EnumValueDescriptor) value).getNumber(); + } else if (field.getType() == FieldDescriptor.Type.MESSAGE) { + if (value != null && !metadata.defaultValue.getClass().isInstance(value)) { + // The value is not the exact right message type. However, if it + // is an alternative implementation of the same type -- e.g. a + // DynamicMessage -- we should accept it. In this case we can make + // a copy of the message. + value = + ((Message) metadata.defaultValue).toBuilder().mergeFrom((Message) value).build(); + } } setValue((V) value); } diff --git a/java/core/src/main/java/com/google/protobuf/NioByteString.java b/java/core/src/main/java/com/google/protobuf/NioByteString.java index 6163c7b1..76594809 100644 --- a/java/core/src/main/java/com/google/protobuf/NioByteString.java +++ b/java/core/src/main/java/com/google/protobuf/NioByteString.java @@ -30,6 +30,8 @@ package com.google.protobuf; +import static com.google.protobuf.Internal.checkNotNull; + import java.io.IOException; import java.io.InputStream; import java.io.InvalidObjectException; @@ -49,10 +51,9 @@ final class NioByteString extends ByteString.LeafByteString { private final ByteBuffer buffer; NioByteString(ByteBuffer buffer) { - if (buffer == null) { - throw new NullPointerException("buffer"); - } + checkNotNull(buffer, "buffer"); + // Use native byte order for fast fixed32/64 operations. this.buffer = buffer.slice().order(ByteOrder.nativeOrder()); } @@ -266,7 +267,7 @@ final class NioByteString extends ByteString.LeafByteString { @Override public CodedInputStream newCodedInput() { - return CodedInputStream.newInstance(buffer); + return CodedInputStream.newInstance(buffer, true); } /** diff --git a/java/core/src/main/java/com/google/protobuf/RopeByteString.java b/java/core/src/main/java/com/google/protobuf/RopeByteString.java index 3f3e9bd1..6fa555df 100644 --- a/java/core/src/main/java/com/google/protobuf/RopeByteString.java +++ b/java/core/src/main/java/com/google/protobuf/RopeByteString.java @@ -406,6 +406,7 @@ final class RopeByteString extends ByteString { right.writeTo(output); } + @Override protected String toStringInternal(Charset charset) { return new String(toByteArray(), charset); diff --git a/java/core/src/main/java/com/google/protobuf/TextFormat.java b/java/core/src/main/java/com/google/protobuf/TextFormat.java index ff13675d..49708242 100644 --- a/java/core/src/main/java/com/google/protobuf/TextFormat.java +++ b/java/core/src/main/java/com/google/protobuf/TextFormat.java @@ -1576,14 +1576,22 @@ public final class TextFormat { // Support specifying repeated field values as a comma-separated list. // Ex."foo: [1, 2, 3]" if (field.isRepeated() && tokenizer.tryConsume("[")) { - while (true) { - consumeFieldValue(tokenizer, extensionRegistry, target, field, extension, - parseTreeBuilder, unknownFields); - if (tokenizer.tryConsume("]")) { - // End of list. - break; + if (!tokenizer.tryConsume("]")) { // Allow "foo: []" to be treated as empty. + while (true) { + consumeFieldValue( + tokenizer, + extensionRegistry, + target, + field, + extension, + parseTreeBuilder, + unknownFields); + if (tokenizer.tryConsume("]")) { + // End of list. + break; + } + tokenizer.consume(","); } - tokenizer.consume(","); } } else { consumeFieldValue(tokenizer, extensionRegistry, target, field, diff --git a/java/core/src/main/java/com/google/protobuf/TextFormatParseInfoTree.java b/java/core/src/main/java/com/google/protobuf/TextFormatParseInfoTree.java index 5c43b2c3..0127ce92 100644 --- a/java/core/src/main/java/com/google/protobuf/TextFormatParseInfoTree.java +++ b/java/core/src/main/java/com/google/protobuf/TextFormatParseInfoTree.java @@ -45,7 +45,8 @@ import java.util.Map.Entry; * * <p>The locations of primary fields values are retrieved by {@code getLocation} or * {@code getLocations}. The locations of sub message values are within nested - * {@code TextFormatParseInfoTree}s and are retrieve by {@code getNestedTree} or {@code getNestedTrees}. + * {@code TextFormatParseInfoTree}s and are retrieve by {@code getNestedTree} or + * {@code getNestedTrees}. * * <p>The {@code TextFormatParseInfoTree} is created by a Builder. */ diff --git a/java/core/src/main/java/com/google/protobuf/UnsafeByteOperations.java b/java/core/src/main/java/com/google/protobuf/UnsafeByteOperations.java index e72a6b4d..878c7758 100644 --- a/java/core/src/main/java/com/google/protobuf/UnsafeByteOperations.java +++ b/java/core/src/main/java/com/google/protobuf/UnsafeByteOperations.java @@ -67,16 +67,34 @@ public final class UnsafeByteOperations { /** * An unsafe operation that returns a {@link ByteString} that is backed by the provided buffer. * + * @param buffer the buffer to be wrapped + * @return a {@link ByteString} backed by the provided buffer + */ + public static ByteString unsafeWrap(byte[] buffer) { + return ByteString.wrap(buffer); + } + + /** + * An unsafe operation that returns a {@link ByteString} that is backed by a subregion of the + * provided buffer. + * + * @param buffer the buffer to be wrapped + * @param offset the offset of the wrapped region + * @param length the number of bytes of the wrapped region + * @return a {@link ByteString} backed by the provided buffer + */ + public static ByteString unsafeWrap(byte[] buffer, int offset, int length) { + return ByteString.wrap(buffer, offset, length); + } + + /** + * An unsafe operation that returns a {@link ByteString} that is backed by the provided buffer. + * * @param buffer the Java NIO buffer to be wrapped * @return a {@link ByteString} backed by the provided buffer */ public static ByteString unsafeWrap(ByteBuffer buffer) { - if (buffer.hasArray()) { - final int offset = buffer.arrayOffset(); - return ByteString.wrap(buffer.array(), offset + buffer.position(), buffer.remaining()); - } else { - return new NioByteString(buffer); - } + return ByteString.wrap(buffer); } /** @@ -98,4 +116,5 @@ public final class UnsafeByteOperations { public static void unsafeWriteTo(ByteString bytes, ByteOutput output) throws IOException { bytes.writeTo(output); } + } diff --git a/java/core/src/main/java/com/google/protobuf/UnsafeUtil.java b/java/core/src/main/java/com/google/protobuf/UnsafeUtil.java index 6a4787d1..5f7bafd6 100644 --- a/java/core/src/main/java/com/google/protobuf/UnsafeUtil.java +++ b/java/core/src/main/java/com/google/protobuf/UnsafeUtil.java @@ -30,17 +30,14 @@ package com.google.protobuf; -import sun.misc.Unsafe; - import java.lang.reflect.Field; import java.nio.Buffer; import java.nio.ByteBuffer; import java.security.AccessController; import java.security.PrivilegedExceptionAction; +import sun.misc.Unsafe; -/** - * Utility class for working with unsafe operations. - */ +/** Utility class for working with unsafe operations. */ // TODO(nathanmittler): Add support for Android Memory/MemoryBlock final class UnsafeUtil { private static final sun.misc.Unsafe UNSAFE = getUnsafe(); @@ -50,8 +47,7 @@ final class UnsafeUtil { private static final long ARRAY_BASE_OFFSET = byteArrayBaseOffset(); private static final long BUFFER_ADDRESS_OFFSET = fieldOffset(field(Buffer.class, "address")); - private UnsafeUtil() { - } + private UnsafeUtil() {} static boolean hasUnsafeArrayOperations() { return HAS_UNSAFE_ARRAY_OPERATIONS; @@ -61,27 +57,83 @@ final class UnsafeUtil { return HAS_UNSAFE_BYTEBUFFER_OPERATIONS; } + static Object allocateInstance(Class<?> clazz) { + try { + return UNSAFE.allocateInstance(clazz); + } catch (InstantiationException e) { + throw new RuntimeException(e); + } + } + + static long objectFieldOffset(Field field) { + return UNSAFE.objectFieldOffset(field); + } + static long getArrayBaseOffset() { return ARRAY_BASE_OFFSET; } - static byte getByte(byte[] target, long offset) { + static byte getByte(Object target, long offset) { return UNSAFE.getByte(target, offset); } - static void putByte(byte[] target, long offset, byte value) { + static void putByte(Object target, long offset, byte value) { UNSAFE.putByte(target, offset, value); } - static void copyMemory( - byte[] src, long srcOffset, byte[] target, long targetOffset, long length) { - UNSAFE.copyMemory(src, srcOffset, target, targetOffset, length); + static int getInt(Object target, long offset) { + return UNSAFE.getInt(target, offset); } - static long getLong(byte[] target, long offset) { + static void putInt(Object target, long offset, int value) { + UNSAFE.putInt(target, offset, value); + } + + static long getLong(Object target, long offset) { return UNSAFE.getLong(target, offset); } + static void putLong(Object target, long offset, long value) { + UNSAFE.putLong(target, offset, value); + } + + static boolean getBoolean(Object target, long offset) { + return UNSAFE.getBoolean(target, offset); + } + + static void putBoolean(Object target, long offset, boolean value) { + UNSAFE.putBoolean(target, offset, value); + } + + static float getFloat(Object target, long offset) { + return UNSAFE.getFloat(target, offset); + } + + static void putFloat(Object target, long offset, float value) { + UNSAFE.putFloat(target, offset, value); + } + + static double getDouble(Object target, long offset) { + return UNSAFE.getDouble(target, offset); + } + + static void putDouble(Object target, long offset, double value) { + UNSAFE.putDouble(target, offset, value); + } + + static Object getObject(Object target, long offset) { + return UNSAFE.getObject(target, offset); + } + + static void putObject(Object target, long offset, Object value) { + UNSAFE.putObject(target, offset, value); + } + + static void copyMemory( + Object src, long srcOffset, Object target, long targetOffset, long length) { + UNSAFE.copyMemory(src, srcOffset, target, targetOffset, length); + } + static byte getByte(long address) { return UNSAFE.getByte(address); } @@ -90,14 +142,30 @@ final class UnsafeUtil { UNSAFE.putByte(address, value); } + static int getInt(long address) { + return UNSAFE.getInt(address); + } + + static void putInt(long address, int value) { + UNSAFE.putInt(address, value); + } + static long getLong(long address) { return UNSAFE.getLong(address); } + static void putLong(long address, long value) { + UNSAFE.putLong(address, value); + } + static void copyMemory(long srcAddress, long targetAddress, long length) { UNSAFE.copyMemory(srcAddress, targetAddress, length); } + static void setMemory(long address, long numBytes, byte value) { + UNSAFE.setMemory(address, numBytes, value); + } + /** * Gets the offset of the {@code address} field of the given direct {@link ByteBuffer}. */ @@ -136,18 +204,29 @@ final class UnsafeUtil { return unsafe; } - /** - * Indicates whether or not unsafe array operations are supported on this platform. - */ + /** Indicates whether or not unsafe array operations are supported on this platform. */ private static boolean supportsUnsafeArrayOperations() { boolean supported = false; if (UNSAFE != null) { try { Class<?> clazz = UNSAFE.getClass(); + clazz.getMethod("objectFieldOffset", Field.class); + clazz.getMethod("allocateInstance", Class.class); clazz.getMethod("arrayBaseOffset", Class.class); clazz.getMethod("getByte", Object.class, long.class); clazz.getMethod("putByte", Object.class, long.class, byte.class); + clazz.getMethod("getBoolean", Object.class, long.class); + clazz.getMethod("putBoolean", Object.class, long.class, boolean.class); + clazz.getMethod("getInt", Object.class, long.class); + clazz.getMethod("putInt", Object.class, long.class, int.class); clazz.getMethod("getLong", Object.class, long.class); + clazz.getMethod("putLong", Object.class, long.class, long.class); + clazz.getMethod("getFloat", Object.class, long.class); + clazz.getMethod("putFloat", Object.class, long.class, float.class); + clazz.getMethod("getDouble", Object.class, long.class); + clazz.getMethod("putDouble", Object.class, long.class, double.class); + clazz.getMethod("getObject", Object.class, long.class); + clazz.getMethod("putObject", Object.class, long.class, Object.class); clazz.getMethod( "copyMemory", Object.class, long.class, Object.class, long.class, long.class); supported = true; @@ -163,11 +242,17 @@ final class UnsafeUtil { if (UNSAFE != null) { try { Class<?> clazz = UNSAFE.getClass(); + // Methods for getting direct buffer address. clazz.getMethod("objectFieldOffset", Field.class); - clazz.getMethod("getByte", long.class); clazz.getMethod("getLong", Object.class, long.class); + + clazz.getMethod("getByte", long.class); clazz.getMethod("putByte", long.class, byte.class); + clazz.getMethod("getInt", long.class); + clazz.getMethod("putInt", long.class, int.class); clazz.getMethod("getLong", long.class); + clazz.getMethod("putLong", long.class, long.class); + clazz.getMethod("setMemory", long.class, long.class, byte.class); clazz.getMethod("copyMemory", long.class, long.class, long.class); supported = true; } catch (Throwable e) { diff --git a/java/core/src/main/java/com/google/protobuf/WireFormat.java b/java/core/src/main/java/com/google/protobuf/WireFormat.java index 6a58b081..0b0cdb7d 100644 --- a/java/core/src/main/java/com/google/protobuf/WireFormat.java +++ b/java/core/src/main/java/com/google/protobuf/WireFormat.java @@ -47,6 +47,10 @@ public final class WireFormat { // Do not allow instantiation. private WireFormat() {} + static final int FIXED_32_SIZE = 4; + static final int FIXED_64_SIZE = 8; + static final int MAX_VARINT_SIZE = 10; + public static final int WIRETYPE_VARINT = 0; public static final int WIRETYPE_FIXED64 = 1; public static final int WIRETYPE_LENGTH_DELIMITED = 2; diff --git a/java/core/src/test/java/com/google/protobuf/AbstractMessageTest.java b/java/core/src/test/java/com/google/protobuf/AbstractMessageTest.java index 7dc9fc15..622e36a4 100644 --- a/java/core/src/test/java/com/google/protobuf/AbstractMessageTest.java +++ b/java/core/src/test/java/com/google/protobuf/AbstractMessageTest.java @@ -40,9 +40,8 @@ import protobuf_unittest.UnittestProto.TestPackedTypes; import protobuf_unittest.UnittestProto.TestRequired; import protobuf_unittest.UnittestProto.TestRequiredForeign; import protobuf_unittest.UnittestProto.TestUnpackedTypes; -import junit.framework.TestCase; - import java.util.Map; +import junit.framework.TestCase; /** * Unit test for {@link AbstractMessage}. @@ -492,7 +491,6 @@ public class AbstractMessageTest extends TestCase { checkEqualsIsConsistent(eUnknownFields, eUnknownFields2); } - /** * Asserts that the given proto has symmetric equals and hashCode methods. */ diff --git a/java/core/src/test/java/com/google/protobuf/BooleanArrayListTest.java b/java/core/src/test/java/com/google/protobuf/BooleanArrayListTest.java index ec139225..18132e9e 100644 --- a/java/core/src/test/java/com/google/protobuf/BooleanArrayListTest.java +++ b/java/core/src/test/java/com/google/protobuf/BooleanArrayListTest.java @@ -32,11 +32,10 @@ package com.google.protobuf; import static java.util.Arrays.asList; -import junit.framework.TestCase; - import java.util.Collections; import java.util.ConcurrentModificationException; import java.util.Iterator; +import junit.framework.TestCase; /** * Tests for {@link BooleanArrayList}. diff --git a/java/core/src/test/java/com/google/protobuf/ByteBufferWriterTest.java b/java/core/src/test/java/com/google/protobuf/ByteBufferWriterTest.java index cbe742e5..6b1cfe78 100644 --- a/java/core/src/test/java/com/google/protobuf/ByteBufferWriterTest.java +++ b/java/core/src/test/java/com/google/protobuf/ByteBufferWriterTest.java @@ -30,13 +30,12 @@ package com.google.protobuf; -import junit.framework.TestCase; - import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Random; +import junit.framework.TestCase; /** * Tests for {@link ByteBufferWriter}. diff --git a/java/core/src/test/java/com/google/protobuf/ByteStringTest.java b/java/core/src/test/java/com/google/protobuf/ByteStringTest.java index ad9f266e..be71f1f5 100644 --- a/java/core/src/test/java/com/google/protobuf/ByteStringTest.java +++ b/java/core/src/test/java/com/google/protobuf/ByteStringTest.java @@ -31,9 +31,6 @@ package com.google.protobuf; import com.google.protobuf.ByteString.Output; - -import junit.framework.TestCase; - import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -48,6 +45,7 @@ import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; import java.util.Random; +import junit.framework.TestCase; /** * Test methods with implementations in {@link ByteString}, plus do some top-level "integration" diff --git a/java/core/src/test/java/com/google/protobuf/CodedInputStreamTest.java b/java/core/src/test/java/com/google/protobuf/CodedInputStreamTest.java index ca940ced..e2ab0df9 100644 --- a/java/core/src/test/java/com/google/protobuf/CodedInputStreamTest.java +++ b/java/core/src/test/java/com/google/protobuf/CodedInputStreamTest.java @@ -30,20 +30,20 @@ package com.google.protobuf; +import static org.junit.Assert.assertArrayEquals; + import protobuf_unittest.UnittestProto.BoolMessage; import protobuf_unittest.UnittestProto.Int32Message; import protobuf_unittest.UnittestProto.Int64Message; import protobuf_unittest.UnittestProto.TestAllTypes; import protobuf_unittest.UnittestProto.TestRecursiveMessage; - -import junit.framework.TestCase; - import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; +import junit.framework.TestCase; /** * Unit test for {@link CodedInputStream}. @@ -51,10 +51,45 @@ import java.nio.ByteBuffer; * @author kenton@google.com Kenton Varda */ public class CodedInputStreamTest extends TestCase { + private enum InputType { + ARRAY { + @Override + CodedInputStream newDecoder(byte[] data, int blockSize) { + return CodedInputStream.newInstance(data); + } + }, + NIO_HEAP { + @Override + CodedInputStream newDecoder(byte[] data, int blockSize) { + return CodedInputStream.newInstance(ByteBuffer.wrap(data)); + } + }, + NIO_DIRECT { + @Override + CodedInputStream newDecoder(byte[] data, int blockSize) { + ByteBuffer buffer = ByteBuffer.allocateDirect(data.length); + buffer.put(data); + buffer.flip(); + return CodedInputStream.newInstance(buffer); + } + }, + STREAM { + @Override + CodedInputStream newDecoder(byte[] data, int blockSize) { + return CodedInputStream.newInstance(new SmallBlockInputStream(data, blockSize)); + } + }; + + CodedInputStream newDecoder(byte[] data) { + return newDecoder(data, data.length); + } + + abstract CodedInputStream newDecoder(byte[] data, int blockSize); + } + /** - * Helper to construct a byte array from a bunch of bytes. The inputs are - * actually ints so that I can use hex notation and not get stupid errors - * about precision. + * Helper to construct a byte array from a bunch of bytes. The inputs are actually ints so that I + * can use hex notation and not get stupid errors about precision. */ private byte[] bytes(int... bytesAsInts) { byte[] bytes = new byte[bytesAsInts.length]; @@ -65,19 +100,14 @@ public class CodedInputStreamTest extends TestCase { } /** - * An InputStream which limits the number of bytes it reads at a time. - * We use this to make sure that CodedInputStream doesn't screw up when - * reading in small blocks. + * An InputStream which limits the number of bytes it reads at a time. We use this to make sure + * that CodedInputStream doesn't screw up when reading in small blocks. */ private static final class SmallBlockInputStream extends FilterInputStream { private final int blockSize; public SmallBlockInputStream(byte[] data, int blockSize) { - this(new ByteArrayInputStream(data), blockSize); - } - - public SmallBlockInputStream(InputStream in, int blockSize) { - super(in); + super(new ByteArrayInputStream(data)); this.blockSize = blockSize; } @@ -92,54 +122,36 @@ public class CodedInputStreamTest extends TestCase { } } - private void assertDataConsumed(byte[] data, CodedInputStream input) + private void assertDataConsumed(String msg, byte[] data, CodedInputStream input) throws IOException { - assertEquals(data.length, input.getTotalBytesRead()); - assertTrue(input.isAtEnd()); + assertEquals(msg, data.length, input.getTotalBytesRead()); + assertTrue(msg, input.isAtEnd()); } /** - * Parses the given bytes using readRawVarint32() and readRawVarint64() and - * checks that the result matches the given value. + * Parses the given bytes using readRawVarint32() and readRawVarint64() and checks that the result + * matches the given value. */ private void assertReadVarint(byte[] data, long value) throws Exception { - CodedInputStream input = CodedInputStream.newInstance(data); - assertEquals((int) value, input.readRawVarint32()); - assertDataConsumed(data, input); - - input = CodedInputStream.newInstance(data); - assertEquals(value, input.readRawVarint64()); - assertDataConsumed(data, input); - - input = CodedInputStream.newInstance(data); - assertEquals(value, input.readRawVarint64SlowPath()); - assertDataConsumed(data, input); - - input = CodedInputStream.newInstance(data); - assertTrue(input.skipField(WireFormat.WIRETYPE_VARINT)); - assertDataConsumed(data, input); - - // Try different block sizes. - for (int blockSize = 1; blockSize <= 16; blockSize *= 2) { - input = CodedInputStream.newInstance( - new SmallBlockInputStream(data, blockSize)); - assertEquals((int) value, input.readRawVarint32()); - assertDataConsumed(data, input); - - input = CodedInputStream.newInstance( - new SmallBlockInputStream(data, blockSize)); - assertEquals(value, input.readRawVarint64()); - assertDataConsumed(data, input); - - input = CodedInputStream.newInstance( - new SmallBlockInputStream(data, blockSize)); - assertEquals(value, input.readRawVarint64SlowPath()); - assertDataConsumed(data, input); - - input = CodedInputStream.newInstance( - new SmallBlockInputStream(data, blockSize)); - assertTrue(input.skipField(WireFormat.WIRETYPE_VARINT)); - assertDataConsumed(data, input); + for (InputType inputType : InputType.values()) { + // Try different block sizes. + for (int blockSize = 1; blockSize <= 16; blockSize *= 2) { + CodedInputStream input = inputType.newDecoder(data, blockSize); + assertEquals(inputType.name(), (int) value, input.readRawVarint32()); + assertDataConsumed(inputType.name(), data, input); + + input = inputType.newDecoder(data, blockSize); + assertEquals(inputType.name(), value, input.readRawVarint64()); + assertDataConsumed(inputType.name(), data, input); + + input = inputType.newDecoder(data, blockSize); + assertEquals(inputType.name(), value, input.readRawVarint64SlowPath()); + assertDataConsumed(inputType.name(), data, input); + + input = inputType.newDecoder(data, blockSize); + assertTrue(inputType.name(), input.skipField(WireFormat.WIRETYPE_VARINT)); + assertDataConsumed(inputType.name(), data, input); + } } // Try reading direct from an InputStream. We want to verify that it @@ -153,35 +165,26 @@ public class CodedInputStreamTest extends TestCase { } /** - * Parses the given bytes using readRawVarint32() and readRawVarint64() and - * expects them to fail with an InvalidProtocolBufferException whose - * description matches the given one. + * Parses the given bytes using readRawVarint32() and readRawVarint64() and expects them to fail + * with an InvalidProtocolBufferException whose description matches the given one. */ - private void assertReadVarintFailure( - InvalidProtocolBufferException expected, byte[] data) + private void assertReadVarintFailure(InvalidProtocolBufferException expected, byte[] data) throws Exception { - CodedInputStream input = CodedInputStream.newInstance(data); - try { - input.readRawVarint32(); - fail("Should have thrown an exception."); - } catch (InvalidProtocolBufferException e) { - assertEquals(expected.getMessage(), e.getMessage()); - } - - input = CodedInputStream.newInstance(data); - try { - input.readRawVarint64(); - fail("Should have thrown an exception."); - } catch (InvalidProtocolBufferException e) { - assertEquals(expected.getMessage(), e.getMessage()); - } - - input = CodedInputStream.newInstance(data); - try { - input.readRawVarint64SlowPath(); - fail("Should have thrown an exception."); - } catch (InvalidProtocolBufferException e) { - assertEquals(expected.getMessage(), e.getMessage()); + for (InputType inputType : InputType.values()) { + try { + CodedInputStream input = inputType.newDecoder(data); + input.readRawVarint32(); + fail(inputType.name() + ": Should have thrown an exception."); + } catch (InvalidProtocolBufferException e) { + assertEquals(inputType.name(), expected.getMessage(), e.getMessage()); + } + try { + CodedInputStream input = inputType.newDecoder(data); + input.readRawVarint64(); + fail(inputType.name() + ": Should have thrown an exception."); + } catch (InvalidProtocolBufferException e) { + assertEquals(inputType.name(), expected.getMessage(), e.getMessage()); + } } // Make sure we get the same error when reading direct from an InputStream. @@ -201,72 +204,74 @@ public class CodedInputStreamTest extends TestCase { // 14882 assertReadVarint(bytes(0xa2, 0x74), (0x22 << 0) | (0x74 << 7)); // 2961488830 - assertReadVarint(bytes(0xbe, 0xf7, 0x92, 0x84, 0x0b), - (0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) | - (0x0bL << 28)); + assertReadVarint( + bytes(0xbe, 0xf7, 0x92, 0x84, 0x0b), + (0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) | (0x0bL << 28)); // 64-bit // 7256456126 - assertReadVarint(bytes(0xbe, 0xf7, 0x92, 0x84, 0x1b), - (0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) | - (0x1bL << 28)); + assertReadVarint( + bytes(0xbe, 0xf7, 0x92, 0x84, 0x1b), + (0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) | (0x1bL << 28)); // 41256202580718336 assertReadVarint( - bytes(0x80, 0xe6, 0xeb, 0x9c, 0xc3, 0xc9, 0xa4, 0x49), - (0x00 << 0) | (0x66 << 7) | (0x6b << 14) | (0x1c << 21) | - (0x43L << 28) | (0x49L << 35) | (0x24L << 42) | (0x49L << 49)); + bytes(0x80, 0xe6, 0xeb, 0x9c, 0xc3, 0xc9, 0xa4, 0x49), + (0x00 << 0) + | (0x66 << 7) + | (0x6b << 14) + | (0x1c << 21) + | (0x43L << 28) + | (0x49L << 35) + | (0x24L << 42) + | (0x49L << 49)); // 11964378330978735131 assertReadVarint( - bytes(0x9b, 0xa8, 0xf9, 0xc2, 0xbb, 0xd6, 0x80, 0x85, 0xa6, 0x01), - (0x1b << 0) | (0x28 << 7) | (0x79 << 14) | (0x42 << 21) | - (0x3bL << 28) | (0x56L << 35) | (0x00L << 42) | - (0x05L << 49) | (0x26L << 56) | (0x01L << 63)); + bytes(0x9b, 0xa8, 0xf9, 0xc2, 0xbb, 0xd6, 0x80, 0x85, 0xa6, 0x01), + (0x1b << 0) + | (0x28 << 7) + | (0x79 << 14) + | (0x42 << 21) + | (0x3bL << 28) + | (0x56L << 35) + | (0x00L << 42) + | (0x05L << 49) + | (0x26L << 56) + | (0x01L << 63)); // Failures assertReadVarintFailure( - InvalidProtocolBufferException.malformedVarint(), - bytes(0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, - 0x00)); - assertReadVarintFailure( - InvalidProtocolBufferException.truncatedMessage(), - bytes(0x80)); + InvalidProtocolBufferException.malformedVarint(), + bytes(0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00)); + assertReadVarintFailure(InvalidProtocolBufferException.truncatedMessage(), bytes(0x80)); } /** - * Parses the given bytes using readRawLittleEndian32() and checks - * that the result matches the given value. + * Parses the given bytes using readRawLittleEndian32() and checks that the result matches the + * given value. */ - private void assertReadLittleEndian32(byte[] data, int value) - throws Exception { - CodedInputStream input = CodedInputStream.newInstance(data); - assertEquals(value, input.readRawLittleEndian32()); - assertTrue(input.isAtEnd()); - - // Try different block sizes. - for (int blockSize = 1; blockSize <= 16; blockSize *= 2) { - input = CodedInputStream.newInstance( - new SmallBlockInputStream(data, blockSize)); - assertEquals(value, input.readRawLittleEndian32()); - assertTrue(input.isAtEnd()); + private void assertReadLittleEndian32(byte[] data, int value) throws Exception { + for (InputType inputType : InputType.values()) { + // Try different block sizes. + for (int blockSize = 1; blockSize <= 16; blockSize *= 2) { + CodedInputStream input = inputType.newDecoder(data, blockSize); + assertEquals(inputType.name(), value, input.readRawLittleEndian32()); + assertTrue(inputType.name(), input.isAtEnd()); + } } } /** - * Parses the given bytes using readRawLittleEndian64() and checks - * that the result matches the given value. + * Parses the given bytes using readRawLittleEndian64() and checks that the result matches the + * given value. */ - private void assertReadLittleEndian64(byte[] data, long value) - throws Exception { - CodedInputStream input = CodedInputStream.newInstance(data); - assertEquals(value, input.readRawLittleEndian64()); - assertTrue(input.isAtEnd()); - - // Try different block sizes. - for (int blockSize = 1; blockSize <= 16; blockSize *= 2) { - input = CodedInputStream.newInstance( - new SmallBlockInputStream(data, blockSize)); - assertEquals(value, input.readRawLittleEndian64()); - assertTrue(input.isAtEnd()); + private void assertReadLittleEndian64(byte[] data, long value) throws Exception { + for (InputType inputType : InputType.values()) { + // Try different block sizes. + for (int blockSize = 1; blockSize <= 16; blockSize *= 2) { + CodedInputStream input = inputType.newDecoder(data, blockSize); + assertEquals(inputType.name(), value, input.readRawLittleEndian64()); + assertTrue(inputType.name(), input.isAtEnd()); + } } } @@ -276,40 +281,32 @@ public class CodedInputStreamTest extends TestCase { assertReadLittleEndian32(bytes(0xf0, 0xde, 0xbc, 0x9a), 0x9abcdef0); assertReadLittleEndian64( - bytes(0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12), - 0x123456789abcdef0L); + bytes(0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12), 0x123456789abcdef0L); assertReadLittleEndian64( - bytes(0x78, 0x56, 0x34, 0x12, 0xf0, 0xde, 0xbc, 0x9a), - 0x9abcdef012345678L); + bytes(0x78, 0x56, 0x34, 0x12, 0xf0, 0xde, 0xbc, 0x9a), 0x9abcdef012345678L); } /** Test decodeZigZag32() and decodeZigZag64(). */ public void testDecodeZigZag() throws Exception { - assertEquals( 0, CodedInputStream.decodeZigZag32(0)); + assertEquals(0, CodedInputStream.decodeZigZag32(0)); assertEquals(-1, CodedInputStream.decodeZigZag32(1)); - assertEquals( 1, CodedInputStream.decodeZigZag32(2)); + assertEquals(1, CodedInputStream.decodeZigZag32(2)); assertEquals(-2, CodedInputStream.decodeZigZag32(3)); assertEquals(0x3FFFFFFF, CodedInputStream.decodeZigZag32(0x7FFFFFFE)); assertEquals(0xC0000000, CodedInputStream.decodeZigZag32(0x7FFFFFFF)); assertEquals(0x7FFFFFFF, CodedInputStream.decodeZigZag32(0xFFFFFFFE)); assertEquals(0x80000000, CodedInputStream.decodeZigZag32(0xFFFFFFFF)); - assertEquals( 0, CodedInputStream.decodeZigZag64(0)); + assertEquals(0, CodedInputStream.decodeZigZag64(0)); assertEquals(-1, CodedInputStream.decodeZigZag64(1)); - assertEquals( 1, CodedInputStream.decodeZigZag64(2)); + assertEquals(1, CodedInputStream.decodeZigZag64(2)); assertEquals(-2, CodedInputStream.decodeZigZag64(3)); - assertEquals(0x000000003FFFFFFFL, - CodedInputStream.decodeZigZag64(0x000000007FFFFFFEL)); - assertEquals(0xFFFFFFFFC0000000L, - CodedInputStream.decodeZigZag64(0x000000007FFFFFFFL)); - assertEquals(0x000000007FFFFFFFL, - CodedInputStream.decodeZigZag64(0x00000000FFFFFFFEL)); - assertEquals(0xFFFFFFFF80000000L, - CodedInputStream.decodeZigZag64(0x00000000FFFFFFFFL)); - assertEquals(0x7FFFFFFFFFFFFFFFL, - CodedInputStream.decodeZigZag64(0xFFFFFFFFFFFFFFFEL)); - assertEquals(0x8000000000000000L, - CodedInputStream.decodeZigZag64(0xFFFFFFFFFFFFFFFFL)); + assertEquals(0x000000003FFFFFFFL, CodedInputStream.decodeZigZag64(0x000000007FFFFFFEL)); + assertEquals(0xFFFFFFFFC0000000L, CodedInputStream.decodeZigZag64(0x000000007FFFFFFFL)); + assertEquals(0x000000007FFFFFFFL, CodedInputStream.decodeZigZag64(0x00000000FFFFFFFEL)); + assertEquals(0xFFFFFFFF80000000L, CodedInputStream.decodeZigZag64(0x00000000FFFFFFFFL)); + assertEquals(0x7FFFFFFFFFFFFFFFL, CodedInputStream.decodeZigZag64(0xFFFFFFFFFFFFFFFEL)); + assertEquals(0x8000000000000000L, CodedInputStream.decodeZigZag64(0xFFFFFFFFFFFFFFFFL)); } /** Tests reading and parsing a whole message with every field type. */ @@ -319,14 +316,12 @@ public class CodedInputStreamTest extends TestCase { byte[] rawBytes = message.toByteArray(); assertEquals(rawBytes.length, message.getSerializedSize()); - TestAllTypes message2 = TestAllTypes.parseFrom(rawBytes); - TestUtil.assertAllFieldsSet(message2); - - // Try different block sizes. - for (int blockSize = 1; blockSize < 256; blockSize *= 2) { - message2 = TestAllTypes.parseFrom( - new SmallBlockInputStream(rawBytes, blockSize)); - TestUtil.assertAllFieldsSet(message2); + for (InputType inputType : InputType.values()) { + // Try different block sizes. + for (int blockSize = 1; blockSize < 256; blockSize *= 2) { + TestAllTypes message2 = TestAllTypes.parseFrom(inputType.newDecoder(rawBytes, blockSize)); + TestUtil.assertAllFieldsSet(message2); + } } } @@ -335,57 +330,65 @@ public class CodedInputStreamTest extends TestCase { TestAllTypes message = TestUtil.getAllSet(); byte[] rawBytes = message.toByteArray(); - // Create two parallel inputs. Parse one as unknown fields while using - // skipField() to skip each field on the other. Expect the same tags. - CodedInputStream input1 = CodedInputStream.newInstance(rawBytes); - CodedInputStream input2 = CodedInputStream.newInstance(rawBytes); + InputType[] inputTypes = InputType.values(); + CodedInputStream[] inputs = new CodedInputStream[inputTypes.length]; + for (int i = 0; i < inputs.length; ++i) { + inputs[i] = inputTypes[i].newDecoder(rawBytes); + } UnknownFieldSet.Builder unknownFields = UnknownFieldSet.newBuilder(); while (true) { + CodedInputStream input1 = inputs[0]; int tag = input1.readTag(); - assertEquals(tag, input2.readTag()); + // Ensure that the rest match. + for (int i = 1; i < inputs.length; ++i) { + assertEquals(inputTypes[i].name(), tag, inputs[i].readTag()); + } if (tag == 0) { break; } unknownFields.mergeFieldFrom(tag, input1); - input2.skipField(tag); + // Skip the field for the rest of the inputs. + for (int i = 1; i < inputs.length; ++i) { + inputs[i].skipField(tag); + } } } /** - * Test that a bug in skipRawBytes() has been fixed: if the skip skips - * exactly up to a limit, this should not break things. + * Test that a bug in skipRawBytes() has been fixed: if the skip skips exactly up to a limit, this + * should not break things. */ public void testSkipRawBytesBug() throws Exception { - byte[] rawBytes = new byte[] { 1, 2 }; - CodedInputStream input = CodedInputStream.newInstance(rawBytes); - - int limit = input.pushLimit(1); - input.skipRawBytes(1); - input.popLimit(limit); - assertEquals(2, input.readRawByte()); + byte[] rawBytes = new byte[] {1, 2}; + for (InputType inputType : InputType.values()) { + CodedInputStream input = inputType.newDecoder(rawBytes); + int limit = input.pushLimit(1); + input.skipRawBytes(1); + input.popLimit(limit); + assertEquals(inputType.name(), 2, input.readRawByte()); + } } /** - * Test that a bug in skipRawBytes() has been fixed: if the skip skips - * past the end of a buffer with a limit that has been set past the end of - * that buffer, this should not break things. + * Test that a bug in skipRawBytes() has been fixed: if the skip skips past the end of a buffer + * with a limit that has been set past the end of that buffer, this should not break things. */ public void testSkipRawBytesPastEndOfBufferWithLimit() throws Exception { - byte[] rawBytes = new byte[] { 1, 2, 3, 4, 5 }; - CodedInputStream input = CodedInputStream.newInstance( - new SmallBlockInputStream(rawBytes, 3)); - - int limit = input.pushLimit(4); - // In order to expose the bug we need to read at least one byte to prime the - // buffer inside the CodedInputStream. - assertEquals(1, input.readRawByte()); - // Skip to the end of the limit. - input.skipRawBytes(3); - assertTrue(input.isAtEnd()); - input.popLimit(limit); - assertEquals(5, input.readRawByte()); + byte[] rawBytes = new byte[] {1, 2, 3, 4, 5}; + for (InputType inputType : InputType.values()) { + CodedInputStream input = inputType.newDecoder(rawBytes); + int limit = input.pushLimit(4); + // In order to expose the bug we need to read at least one byte to prime the + // buffer inside the CodedInputStream. + assertEquals(inputType.name(), 1, input.readRawByte()); + // Skip to the end of the limit. + input.skipRawBytes(3); + assertTrue(inputType.name(), input.isAtEnd()); + input.popLimit(limit); + assertEquals(inputType.name(), 5, input.readRawByte()); + } } public void testReadHugeBlob() throws Exception { @@ -401,19 +404,22 @@ public class CodedInputStreamTest extends TestCase { builder.setOptionalBytes(ByteString.copyFrom(blob)); TestAllTypes message = builder.build(); - // Serialize and parse it. Make sure to parse from an InputStream, not - // directly from a ByteString, so that CodedInputStream uses buffered - // reading. - TestAllTypes message2 = - TestAllTypes.parseFrom(message.toByteString().newInput()); - - assertEquals(message.getOptionalBytes(), message2.getOptionalBytes()); - - // Make sure all the other fields were parsed correctly. - TestAllTypes message3 = TestAllTypes.newBuilder(message2) - .setOptionalBytes(TestUtil.getAllSet().getOptionalBytes()) - .build(); - TestUtil.assertAllFieldsSet(message3); + byte[] data = message.toByteArray(); + for (InputType inputType : InputType.values()) { + // Serialize and parse it. Make sure to parse from an InputStream, not + // directly from a ByteString, so that CodedInputStream uses buffered + // reading. + TestAllTypes message2 = TestAllTypes.parseFrom(inputType.newDecoder(data)); + + assertEquals(inputType.name(), message.getOptionalBytes(), message2.getOptionalBytes()); + + // Make sure all the other fields were parsed correctly. + TestAllTypes message3 = + TestAllTypes.newBuilder(message2) + .setOptionalBytes(TestUtil.getAllSet().getOptionalBytes()) + .build(); + TestUtil.assertAllFieldsSet(message3); + } } public void testReadMaliciouslyLargeBlob() throws Exception { @@ -423,17 +429,19 @@ public class CodedInputStreamTest extends TestCase { int tag = WireFormat.makeTag(1, WireFormat.WIRETYPE_LENGTH_DELIMITED); output.writeRawVarint32(tag); output.writeRawVarint32(0x7FFFFFFF); - output.writeRawBytes(new byte[32]); // Pad with a few random bytes. + output.writeRawBytes(new byte[32]); // Pad with a few random bytes. output.flush(); - CodedInputStream input = rawOutput.toByteString().newCodedInput(); - assertEquals(tag, input.readTag()); - - try { - input.readBytes(); - fail("Should have thrown an exception!"); - } catch (InvalidProtocolBufferException e) { - // success. + byte[] data = rawOutput.toByteString().toByteArray(); + for (InputType inputType : InputType.values()) { + CodedInputStream input = inputType.newDecoder(data); + assertEquals(tag, input.readTag()); + try { + input.readBytes(); + fail(inputType.name() + ": Should have thrown an exception!"); + } catch (InvalidProtocolBufferException e) { + // success. + } } } @@ -441,54 +449,55 @@ public class CodedInputStreamTest extends TestCase { if (depth == 0) { return TestRecursiveMessage.newBuilder().setI(5).build(); } else { - return TestRecursiveMessage.newBuilder() - .setA(makeRecursiveMessage(depth - 1)).build(); + return TestRecursiveMessage.newBuilder().setA(makeRecursiveMessage(depth - 1)).build(); } } - private void assertMessageDepth(TestRecursiveMessage message, int depth) { + private void assertMessageDepth(String msg, TestRecursiveMessage message, int depth) { if (depth == 0) { - assertFalse(message.hasA()); - assertEquals(5, message.getI()); + assertFalse(msg, message.hasA()); + assertEquals(msg, 5, message.getI()); } else { - assertTrue(message.hasA()); - assertMessageDepth(message.getA(), depth - 1); + assertTrue(msg, message.hasA()); + assertMessageDepth(msg, message.getA(), depth - 1); } } public void testMaliciousRecursion() throws Exception { - ByteString data100 = makeRecursiveMessage(100).toByteString(); - ByteString data101 = makeRecursiveMessage(101).toByteString(); + byte[] data100 = makeRecursiveMessage(100).toByteArray(); + byte[] data101 = makeRecursiveMessage(101).toByteArray(); - assertMessageDepth(TestRecursiveMessage.parseFrom(data100), 100); + for (InputType inputType : InputType.values()) { + assertMessageDepth( + inputType.name(), TestRecursiveMessage.parseFrom(inputType.newDecoder(data100)), 100); - try { - TestRecursiveMessage.parseFrom(data101); - fail("Should have thrown an exception!"); - } catch (InvalidProtocolBufferException e) { - // success. - } + try { + TestRecursiveMessage.parseFrom(inputType.newDecoder(data101)); + fail("Should have thrown an exception!"); + } catch (InvalidProtocolBufferException e) { + // success. + } - CodedInputStream input = data100.newCodedInput(); - input.setRecursionLimit(8); - try { - TestRecursiveMessage.parseFrom(input); - fail("Should have thrown an exception!"); - } catch (InvalidProtocolBufferException e) { - // success. + CodedInputStream input = inputType.newDecoder(data100); + input.setRecursionLimit(8); + try { + TestRecursiveMessage.parseFrom(input); + fail(inputType.name() + ": Should have thrown an exception!"); + } catch (InvalidProtocolBufferException e) { + // success. + } } } private void checkSizeLimitExceeded(InvalidProtocolBufferException e) { - assertEquals( - InvalidProtocolBufferException.sizeLimitExceeded().getMessage(), - e.getMessage()); + assertEquals(InvalidProtocolBufferException.sizeLimitExceeded().getMessage(), e.getMessage()); } public void testSizeLimit() throws Exception { - CodedInputStream input = CodedInputStream.newInstance( - new SmallBlockInputStream( - TestUtil.getAllSet().toByteString().newInput(), 16)); + // NOTE: Size limit only applies to the stream-backed CIS. + CodedInputStream input = + CodedInputStream.newInstance( + new SmallBlockInputStream(TestUtil.getAllSet().toByteArray(), 16)); input.setSizeLimit(16); try { @@ -500,8 +509,9 @@ public class CodedInputStreamTest extends TestCase { } public void testResetSizeCounter() throws Exception { - CodedInputStream input = CodedInputStream.newInstance( - new SmallBlockInputStream(new byte[256], 8)); + // NOTE: Size limit only applies to the stream-backed CIS. + CodedInputStream input = + CodedInputStream.newInstance(new SmallBlockInputStream(new byte[256], 8)); input.setSizeLimit(16); input.readRawBytes(16); assertEquals(16, input.getTotalBytesRead()); @@ -515,7 +525,7 @@ public class CodedInputStreamTest extends TestCase { input.resetSizeCounter(); assertEquals(0, input.getTotalBytesRead()); - input.readRawByte(); // No exception thrown. + input.readRawByte(); // No exception thrown. input.resetSizeCounter(); assertEquals(0, input.getTotalBytesRead()); input.readRawBytes(16); @@ -523,7 +533,7 @@ public class CodedInputStreamTest extends TestCase { input.resetSizeCounter(); try { - input.readRawBytes(17); // Hits limit again. + input.readRawBytes(17); // Hits limit again. fail("Should have thrown an exception!"); } catch (InvalidProtocolBufferException expected) { checkSizeLimitExceeded(expected); @@ -531,12 +541,12 @@ public class CodedInputStreamTest extends TestCase { } public void testSizeLimitMultipleMessages() throws Exception { + // NOTE: Size limit only applies to the stream-backed CIS. byte[] bytes = new byte[256]; for (int i = 0; i < bytes.length; i++) { bytes[i] = (byte) i; } - CodedInputStream input = CodedInputStream.newInstance( - new SmallBlockInputStream(bytes, 7)); + CodedInputStream input = CodedInputStream.newInstance(new SmallBlockInputStream(bytes, 7)); input.setSizeLimit(16); for (int i = 0; i < 256 / 16; i++) { byte[] message = input.readRawBytes(16); @@ -566,12 +576,13 @@ public class CodedInputStreamTest extends TestCase { output.writeRawBytes(bytes); output.flush(); - CodedInputStream input = - CodedInputStream.newInstance( - new ByteArrayInputStream(rawOutput.toByteString().toByteArray())); - assertEquals(tag, input.readTag()); - String text = input.readString(); - assertEquals(lorem, text); + byte[] rawInput = rawOutput.toByteString().toByteArray(); + for (InputType inputType : InputType.values()) { + CodedInputStream input = inputType.newDecoder(rawInput); + assertEquals(inputType.name(), tag, input.readTag()); + String text = input.readString(); + assertEquals(inputType.name(), lorem, text); + } } public void testReadStringRequireUtf8() throws Exception { @@ -591,18 +602,18 @@ public class CodedInputStreamTest extends TestCase { output.writeRawBytes(bytes); output.flush(); - CodedInputStream input = - CodedInputStream.newInstance( - new ByteArrayInputStream(rawOutput.toByteString().toByteArray())); - assertEquals(tag, input.readTag()); - String text = input.readStringRequireUtf8(); - assertEquals(lorem, text); + byte[] rawInput = rawOutput.toByteString().toByteArray(); + for (InputType inputType : InputType.values()) { + CodedInputStream input = inputType.newDecoder(rawInput); + assertEquals(inputType.name(), tag, input.readTag()); + String text = input.readStringRequireUtf8(); + assertEquals(inputType.name(), lorem, text); + } } /** - * Tests that if we readString invalid UTF-8 bytes, no exception - * is thrown. Instead, the invalid bytes are replaced with the Unicode - * "replacement character" U+FFFD. + * Tests that if we readString invalid UTF-8 bytes, no exception is thrown. Instead, the invalid + * bytes are replaced with the Unicode "replacement character" U+FFFD. */ public void testReadStringInvalidUtf8() throws Exception { ByteString.Output rawOutput = ByteString.newOutput(); @@ -611,18 +622,21 @@ public class CodedInputStreamTest extends TestCase { int tag = WireFormat.makeTag(1, WireFormat.WIRETYPE_LENGTH_DELIMITED); output.writeRawVarint32(tag); output.writeRawVarint32(1); - output.writeRawBytes(new byte[] { (byte) 0x80 }); + output.writeRawBytes(new byte[] {(byte) 0x80}); output.flush(); - CodedInputStream input = rawOutput.toByteString().newCodedInput(); - assertEquals(tag, input.readTag()); - String text = input.readString(); - assertEquals(0xfffd, text.charAt(0)); + byte[] rawInput = rawOutput.toByteString().toByteArray(); + for (InputType inputType : InputType.values()) { + CodedInputStream input = inputType.newDecoder(rawInput); + assertEquals(inputType.name(), tag, input.readTag()); + String text = input.readString(); + assertEquals(inputType.name(), 0xfffd, text.charAt(0)); + } } /** - * Tests that if we readStringRequireUtf8 invalid UTF-8 bytes, an - * InvalidProtocolBufferException is thrown. + * Tests that if we readStringRequireUtf8 invalid UTF-8 bytes, an InvalidProtocolBufferException + * is thrown. */ public void testReadStringRequireUtf8InvalidUtf8() throws Exception { ByteString.Output rawOutput = ByteString.newOutput(); @@ -631,16 +645,20 @@ public class CodedInputStreamTest extends TestCase { int tag = WireFormat.makeTag(1, WireFormat.WIRETYPE_LENGTH_DELIMITED); output.writeRawVarint32(tag); output.writeRawVarint32(1); - output.writeRawBytes(new byte[] { (byte) 0x80 }); + output.writeRawBytes(new byte[] {(byte) 0x80}); output.flush(); - CodedInputStream input = rawOutput.toByteString().newCodedInput(); - assertEquals(tag, input.readTag()); - try { - input.readStringRequireUtf8(); - fail("Expected invalid UTF-8 exception."); - } catch (InvalidProtocolBufferException exception) { - assertEquals("Protocol message had invalid UTF-8.", exception.getMessage()); + byte[] rawInput = rawOutput.toByteString().toByteArray(); + for (InputType inputType : InputType.values()) { + CodedInputStream input = inputType.newDecoder(rawInput); + assertEquals(tag, input.readTag()); + try { + input.readStringRequireUtf8(); + fail(inputType.name() + ": Expected invalid UTF-8 exception."); + } catch (InvalidProtocolBufferException exception) { + assertEquals( + inputType.name(), "Protocol message had invalid UTF-8.", exception.getMessage()); + } } } @@ -660,13 +678,17 @@ public class CodedInputStreamTest extends TestCase { public void testInvalidTag() throws Exception { // Any tag number which corresponds to field number zero is invalid and // should throw InvalidProtocolBufferException. - for (int i = 0; i < 8; i++) { - try { - CodedInputStream.newInstance(bytes(i)).readTag(); - fail("Should have thrown an exception."); - } catch (InvalidProtocolBufferException e) { - assertEquals(InvalidProtocolBufferException.invalidTag().getMessage(), - e.getMessage()); + for (InputType inputType : InputType.values()) { + for (int i = 0; i < 8; i++) { + try { + inputType.newDecoder(bytes(i)).readTag(); + fail(inputType.name() + ": Should have thrown an exception."); + } catch (InvalidProtocolBufferException e) { + assertEquals( + inputType.name(), + InvalidProtocolBufferException.invalidTag().getMessage(), + e.getMessage()); + } } } } @@ -678,10 +700,10 @@ public class CodedInputStreamTest extends TestCase { output.writeRawVarint32(0); // One one-byte bytes field output.writeRawVarint32(1); - output.writeRawBytes(new byte[] { (byte) 23 }); + output.writeRawBytes(new byte[] {(byte) 23}); // Another one-byte bytes field output.writeRawVarint32(1); - output.writeRawBytes(new byte[] { (byte) 45 }); + output.writeRawBytes(new byte[] {(byte) 45}); // A bytes field large enough that won't fit into the 4K buffer. final int bytesLength = 16 * 1024; byte[] bytes = new byte[bytesLength]; @@ -691,20 +713,24 @@ public class CodedInputStreamTest extends TestCase { output.writeRawBytes(bytes); output.flush(); - CodedInputStream inputStream = rawOutput.toByteString().newCodedInput(); - - byte[] result = inputStream.readByteArray(); - assertEquals(0, result.length); - result = inputStream.readByteArray(); - assertEquals(1, result.length); - assertEquals((byte) 23, result[0]); - result = inputStream.readByteArray(); - assertEquals(1, result.length); - assertEquals((byte) 45, result[0]); - result = inputStream.readByteArray(); - assertEquals(bytesLength, result.length); - assertEquals((byte) 67, result[0]); - assertEquals((byte) 89, result[bytesLength - 1]); + + byte[] rawInput = rawOutput.toByteString().toByteArray(); + for (InputType inputType : InputType.values()) { + CodedInputStream inputStream = inputType.newDecoder(rawInput); + + byte[] result = inputStream.readByteArray(); + assertEquals(inputType.name(), 0, result.length); + result = inputStream.readByteArray(); + assertEquals(inputType.name(), 1, result.length); + assertEquals(inputType.name(), (byte) 23, result[0]); + result = inputStream.readByteArray(); + assertEquals(inputType.name(), 1, result.length); + assertEquals(inputType.name(), (byte) 45, result[0]); + result = inputStream.readByteArray(); + assertEquals(inputType.name(), bytesLength, result.length); + assertEquals(inputType.name(), (byte) 67, result[0]); + assertEquals(inputType.name(), (byte) 89, result[bytesLength - 1]); + } } public void testReadByteBuffer() throws Exception { @@ -714,10 +740,10 @@ public class CodedInputStreamTest extends TestCase { output.writeRawVarint32(0); // One one-byte bytes field output.writeRawVarint32(1); - output.writeRawBytes(new byte[]{(byte) 23}); + output.writeRawBytes(new byte[] {(byte) 23}); // Another one-byte bytes field output.writeRawVarint32(1); - output.writeRawBytes(new byte[]{(byte) 45}); + output.writeRawBytes(new byte[] {(byte) 45}); // A bytes field large enough that won't fit into the 4K buffer. final int bytesLength = 16 * 1024; byte[] bytes = new byte[bytesLength]; @@ -727,21 +753,25 @@ public class CodedInputStreamTest extends TestCase { output.writeRawBytes(bytes); output.flush(); - CodedInputStream inputStream = rawOutput.toByteString().newCodedInput(); - - ByteBuffer result = inputStream.readByteBuffer(); - assertEquals(0, result.capacity()); - result = inputStream.readByteBuffer(); - assertEquals(1, result.capacity()); - assertEquals((byte) 23, result.get()); - result = inputStream.readByteBuffer(); - assertEquals(1, result.capacity()); - assertEquals((byte) 45, result.get()); - result = inputStream.readByteBuffer(); - assertEquals(bytesLength, result.capacity()); - assertEquals((byte) 67, result.get()); - result.position(bytesLength - 1); - assertEquals((byte) 89, result.get()); + + byte[] rawInput = rawOutput.toByteString().toByteArray(); + for (InputType inputType : InputType.values()) { + CodedInputStream inputStream = inputType.newDecoder(rawInput); + + ByteBuffer result = inputStream.readByteBuffer(); + assertEquals(inputType.name(), 0, result.capacity()); + result = inputStream.readByteBuffer(); + assertEquals(inputType.name(), 1, result.capacity()); + assertEquals(inputType.name(), (byte) 23, result.get()); + result = inputStream.readByteBuffer(); + assertEquals(inputType.name(), 1, result.capacity()); + assertEquals(inputType.name(), (byte) 45, result.get()); + result = inputStream.readByteBuffer(); + assertEquals(inputType.name(), bytesLength, result.capacity()); + assertEquals(inputType.name(), (byte) 67, result.get()); + result.position(bytesLength - 1); + assertEquals(inputType.name(), (byte) 89, result.get()); + } } public void testReadByteBufferAliasing() throws Exception { @@ -751,10 +781,10 @@ public class CodedInputStreamTest extends TestCase { output.writeRawVarint32(0); // One one-byte bytes field output.writeRawVarint32(1); - output.writeRawBytes(new byte[]{(byte) 23}); + output.writeRawBytes(new byte[] {(byte) 23}); // Another one-byte bytes field output.writeRawVarint32(1); - output.writeRawBytes(new byte[]{(byte) 45}); + output.writeRawBytes(new byte[] {(byte) 45}); // A bytes field large enough that won't fit into the 4K buffer. final int bytesLength = 16 * 1024; byte[] bytes = new byte[bytesLength]; @@ -763,59 +793,105 @@ public class CodedInputStreamTest extends TestCase { output.writeRawVarint32(bytesLength); output.writeRawBytes(bytes); output.flush(); + byte[] data = byteArrayStream.toByteArray(); - // Without aliasing - CodedInputStream inputStream = CodedInputStream.newInstance(data); - ByteBuffer result = inputStream.readByteBuffer(); - assertEquals(0, result.capacity()); - result = inputStream.readByteBuffer(); - assertTrue(result.array() != data); - assertEquals(1, result.capacity()); - assertEquals((byte) 23, result.get()); - result = inputStream.readByteBuffer(); - assertTrue(result.array() != data); - assertEquals(1, result.capacity()); - assertEquals((byte) 45, result.get()); - result = inputStream.readByteBuffer(); - assertTrue(result.array() != data); - assertEquals(bytesLength, result.capacity()); - assertEquals((byte) 67, result.get()); - result.position(bytesLength - 1); - assertEquals((byte) 89, result.get()); - - // Enable aliasing - inputStream = CodedInputStream.newInstance(data); - inputStream.enableAliasing(true); - result = inputStream.readByteBuffer(); - assertEquals(0, result.capacity()); - result = inputStream.readByteBuffer(); - assertTrue(result.array() == data); - assertEquals(1, result.capacity()); - assertEquals((byte) 23, result.get()); - result = inputStream.readByteBuffer(); - assertTrue(result.array() == data); - assertEquals(1, result.capacity()); - assertEquals((byte) 45, result.get()); - result = inputStream.readByteBuffer(); - assertTrue(result.array() == data); - assertEquals(bytesLength, result.capacity()); - assertEquals((byte) 67, result.get()); - result.position(bytesLength - 1); - assertEquals((byte) 89, result.get()); + for (InputType inputType : InputType.values()) { + if (inputType == InputType.STREAM) { + // Aliasing doesn't apply to stream-backed CIS. + continue; + } + + // Without aliasing + CodedInputStream inputStream = inputType.newDecoder(data); + ByteBuffer result = inputStream.readByteBuffer(); + assertEquals(inputType.name(), 0, result.capacity()); + result = inputStream.readByteBuffer(); + assertTrue(inputType.name(), result.array() != data); + assertEquals(inputType.name(), 1, result.capacity()); + assertEquals(inputType.name(), (byte) 23, result.get()); + result = inputStream.readByteBuffer(); + assertTrue(inputType.name(), result.array() != data); + assertEquals(inputType.name(), 1, result.capacity()); + assertEquals(inputType.name(), (byte) 45, result.get()); + result = inputStream.readByteBuffer(); + assertTrue(inputType.name(), result.array() != data); + assertEquals(inputType.name(), bytesLength, result.capacity()); + assertEquals(inputType.name(), (byte) 67, result.get()); + result.position(bytesLength - 1); + assertEquals(inputType.name(), (byte) 89, result.get()); + + // Enable aliasing + inputStream = inputType.newDecoder(data); + inputStream.enableAliasing(true); + result = inputStream.readByteBuffer(); + assertEquals(inputType.name(), 0, result.capacity()); + result = inputStream.readByteBuffer(); + if (result.hasArray()) { + assertTrue(inputType.name(), result.array() == data); + } + assertEquals(inputType.name(), 1, result.capacity()); + assertEquals(inputType.name(), (byte) 23, result.get()); + result = inputStream.readByteBuffer(); + if (result.hasArray()) { + assertTrue(inputType.name(), result.array() == data); + } + assertEquals(inputType.name(), 1, result.capacity()); + assertEquals(inputType.name(), (byte) 45, result.get()); + result = inputStream.readByteBuffer(); + if (result.hasArray()) { + assertTrue(inputType.name(), result.array() == data); + } + assertEquals(inputType.name(), bytesLength, result.capacity()); + assertEquals(inputType.name(), (byte) 67, result.get()); + result.position(bytesLength - 1); + assertEquals(inputType.name(), (byte) 89, result.get()); + } } public void testCompatibleTypes() throws Exception { long data = 0x100000000L; Int64Message message = Int64Message.newBuilder().setData(data).build(); - ByteString serialized = message.toByteString(); + byte[] serialized = message.toByteArray(); + for (InputType inputType : InputType.values()) { + CodedInputStream inputStream = inputType.newDecoder(serialized); + + // Test int64(long) is compatible with bool(boolean) + BoolMessage msg2 = BoolMessage.parseFrom(inputStream); + assertTrue(msg2.getData()); + + // Test int64(long) is compatible with int32(int) + inputStream = inputType.newDecoder(serialized); + Int32Message msg3 = Int32Message.parseFrom(inputStream); + assertEquals((int) data, msg3.getData()); + } + } - // Test int64(long) is compatible with bool(boolean) - BoolMessage msg2 = BoolMessage.parseFrom(serialized); - assertTrue(msg2.getData()); + public void testSkipInvalidVarint_FastPath() throws Exception { + // Fast path: We have >= 10 bytes available. Ensure we properly recognize a non-ending varint. + byte[] data = new byte[] {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0}; + for (InputType inputType : InputType.values()) { + try { + CodedInputStream input = inputType.newDecoder(data); + input.skipField(WireFormat.makeTag(1, WireFormat.WIRETYPE_VARINT)); + fail(inputType.name() + ": Should have thrown an exception."); + } catch (InvalidProtocolBufferException e) { + // Expected + } + } + } - // Test int64(long) is compatible with int32(int) - Int32Message msg3 = Int32Message.parseFrom(serialized); - assertEquals((int) data, msg3.getData()); + public void testSkipInvalidVarint_SlowPath() throws Exception { + // Slow path: < 10 bytes available. Ensure we properly recognize a non-ending varint. + byte[] data = new byte[] {-1, -1, -1, -1, -1, -1, -1, -1, -1}; + for (InputType inputType : InputType.values()) { + try { + CodedInputStream input = inputType.newDecoder(data); + input.skipField(WireFormat.makeTag(1, WireFormat.WIRETYPE_VARINT)); + fail(inputType.name() + ": Should have thrown an exception."); + } catch (InvalidProtocolBufferException e) { + // Expected + } + } } } diff --git a/java/core/src/test/java/com/google/protobuf/CodedOutputStreamTest.java b/java/core/src/test/java/com/google/protobuf/CodedOutputStreamTest.java index 33aa4357..78f415c2 100644 --- a/java/core/src/test/java/com/google/protobuf/CodedOutputStreamTest.java +++ b/java/core/src/test/java/com/google/protobuf/CodedOutputStreamTest.java @@ -35,15 +35,13 @@ import protobuf_unittest.UnittestProto.SparseEnumMessage; import protobuf_unittest.UnittestProto.TestAllTypes; import protobuf_unittest.UnittestProto.TestPackedTypes; import protobuf_unittest.UnittestProto.TestSparseEnum; - -import junit.framework.TestCase; - import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import junit.framework.TestCase; /** * Unit test for {@link CodedOutputStream}. @@ -151,16 +149,21 @@ public class CodedOutputStreamTest extends TestCase { private final int initialPosition; private final CodedOutputStream stream; private final ByteBuffer buffer; + private final boolean unsafe; - NioDirectCoder(int size) { - this(size, 0); + NioDirectCoder(int size, boolean unsafe) { + this(size, 0, unsafe); } - NioDirectCoder(int size, int initialPosition) { + NioDirectCoder(int size, int initialPosition, boolean unsafe) { + this.unsafe = unsafe; this.initialPosition = initialPosition; buffer = ByteBuffer.allocateDirect(size); buffer.position(initialPosition); - stream = CodedOutputStream.newInstance(buffer); + stream = + unsafe + ? CodedOutputStream.newUnsafeInstance(buffer) + : CodedOutputStream.newSafeInstance(buffer); } @Override @@ -181,7 +184,7 @@ public class CodedOutputStreamTest extends TestCase { @Override public OutputType getOutputType() { - return OutputType.NIO_DIRECT; + return unsafe ? OutputType.NIO_DIRECT_SAFE : OutputType.NIO_DIRECT_UNSAFE; } } @@ -198,10 +201,16 @@ public class CodedOutputStreamTest extends TestCase { return new NioHeapCoder(size); } }, - NIO_DIRECT() { + NIO_DIRECT_SAFE() { @Override Coder newCoder(int size) { - return new NioDirectCoder(size); + return new NioDirectCoder(size, false); + } + }, + NIO_DIRECT_UNSAFE() { + @Override + Coder newCoder(int size) { + return new NioDirectCoder(size, true); } }, STREAM() { @@ -389,6 +398,7 @@ public class CodedOutputStreamTest extends TestCase { != CodedOutputStream.computeUInt32SizeNoTag(string.length() * Utf8.MAX_BYTES_PER_CHAR)); coder.stream().writeStringNoTag(string); + coder.stream().flush(); int stringSize = CodedOutputStream.computeStringSizeNoTag(string); // Verify that the total bytes written is correct @@ -478,11 +488,12 @@ public class CodedOutputStreamTest extends TestCase { public void testWriteByteArrayWithOffsets() throws Exception { byte[] fullArray = bytes(0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88); - byte[] destination = new byte[4]; - CodedOutputStream codedStream = CodedOutputStream.newInstance(destination); - codedStream.writeByteArrayNoTag(fullArray, 2, 2); - assertEqualBytes(OutputType.ARRAY, bytes(0x02, 0x33, 0x44, 0x00), destination); - assertEquals(3, codedStream.getTotalBytesWritten()); + for (OutputType type : new OutputType[] {OutputType.ARRAY}) { + Coder coder = type.newCoder(4); + coder.stream().writeByteArrayNoTag(fullArray, 2, 2); + assertEqualBytes(type, bytes(0x02, 0x33, 0x44), coder.toByteArray()); + assertEquals(3, coder.stream().getTotalBytesWritten()); + } } public void testSerializeUtf8_MultipleSmallWrites() throws Exception { @@ -561,7 +572,12 @@ public class CodedOutputStreamTest extends TestCase { // Tag is one byte, varint describing string length is 1 byte, string length is 9 bytes. // An array of size 1 will cause a failure when trying to write the varint. for (OutputType outputType : - new OutputType[] {OutputType.ARRAY, OutputType.NIO_HEAP, OutputType.NIO_DIRECT}) { + new OutputType[] { + OutputType.ARRAY, + OutputType.NIO_HEAP, + OutputType.NIO_DIRECT_SAFE, + OutputType.NIO_DIRECT_UNSAFE + }) { for (int i = 0; i < 11; i++) { Coder coder = outputType.newCoder(i); try { @@ -599,10 +615,13 @@ public class CodedOutputStreamTest extends TestCase { public void testNioEncodersWithInitialOffsets() throws Exception { String value = "abc"; - for (Coder coder : new Coder[] {new NioHeapCoder(10, 2), new NioDirectCoder(10, 2)}) { + for (Coder coder : + new Coder[] { + new NioHeapCoder(10, 2), new NioDirectCoder(10, 2, false), new NioDirectCoder(10, 2, true) + }) { coder.stream().writeStringNoTag(value); coder.stream().flush(); - assertEqualBytes(coder.getOutputType(), new byte[]{3, 'a', 'b', 'c'}, coder.toByteArray()); + assertEqualBytes(coder.getOutputType(), new byte[] {3, 'a', 'b', 'c'}, coder.toByteArray()); } } diff --git a/java/core/src/test/java/com/google/protobuf/DeprecatedFieldTest.java b/java/core/src/test/java/com/google/protobuf/DeprecatedFieldTest.java index ce85ae2e..2c272a73 100644 --- a/java/core/src/test/java/com/google/protobuf/DeprecatedFieldTest.java +++ b/java/core/src/test/java/com/google/protobuf/DeprecatedFieldTest.java @@ -31,11 +31,9 @@ package com.google.protobuf; import protobuf_unittest.UnittestProto.TestDeprecatedFields; - -import junit.framework.TestCase; - import java.lang.reflect.AnnotatedElement; import java.lang.reflect.Method; +import junit.framework.TestCase; /** * Test field deprecation diff --git a/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java b/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java index b3302441..b60cd620 100644 --- a/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java +++ b/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java @@ -55,15 +55,15 @@ import protobuf_unittest.UnittestProto.ForeignMessage; import protobuf_unittest.UnittestProto.TestAllExtensions; import protobuf_unittest.UnittestProto.TestAllTypes; import protobuf_unittest.UnittestProto.TestExtremeDefaultValues; +import protobuf_unittest.UnittestProto.TestJsonName; import protobuf_unittest.UnittestProto.TestMultipleExtensionRanges; import protobuf_unittest.UnittestProto.TestRequired; import protobuf_unittest.UnittestProto.TestReservedFields; import protobuf_unittest.UnittestProto.TestService; -import junit.framework.TestCase; - import java.util.Arrays; import java.util.Collections; import java.util.List; +import junit.framework.TestCase; /** * Unit test for {@link Descriptors}. @@ -805,4 +805,15 @@ public class DescriptorsTest extends TestCase { Descriptors.FileDescriptor.buildFrom( fileDescriptorProto, new FileDescriptor[0]); } + + public void testFieldJsonName() throws Exception { + Descriptor d = TestJsonName.getDescriptor(); + assertEquals(6, d.getFields().size()); + assertEquals("fieldName1", d.getFields().get(0).getJsonName()); + assertEquals("fieldName2", d.getFields().get(1).getJsonName()); + assertEquals("FieldName3", d.getFields().get(2).getJsonName()); + assertEquals("FieldName4", d.getFields().get(3).getJsonName()); + assertEquals("FIELDNAME5", d.getFields().get(4).getJsonName()); + assertEquals("@type", d.getFields().get(5).getJsonName()); + } } diff --git a/java/core/src/test/java/com/google/protobuf/DoubleArrayListTest.java b/java/core/src/test/java/com/google/protobuf/DoubleArrayListTest.java index 8e8e4fe2..d8942792 100644 --- a/java/core/src/test/java/com/google/protobuf/DoubleArrayListTest.java +++ b/java/core/src/test/java/com/google/protobuf/DoubleArrayListTest.java @@ -32,11 +32,10 @@ package com.google.protobuf; import static java.util.Arrays.asList; -import junit.framework.TestCase; - import java.util.Collections; import java.util.ConcurrentModificationException; import java.util.Iterator; +import junit.framework.TestCase; /** * Tests for {@link DoubleArrayList}. diff --git a/java/core/src/test/java/com/google/protobuf/DynamicMessageTest.java b/java/core/src/test/java/com/google/protobuf/DynamicMessageTest.java index 8f3a74c1..77d14f6b 100644 --- a/java/core/src/test/java/com/google/protobuf/DynamicMessageTest.java +++ b/java/core/src/test/java/com/google/protobuf/DynamicMessageTest.java @@ -37,10 +37,8 @@ import protobuf_unittest.UnittestProto.TestAllExtensions; import protobuf_unittest.UnittestProto.TestAllTypes; import protobuf_unittest.UnittestProto.TestEmptyMessage; import protobuf_unittest.UnittestProto.TestPackedTypes; - -import junit.framework.TestCase; - import java.util.Arrays; +import junit.framework.TestCase; /** * Unit test for {@link DynamicMessage}. See also {@link MessageTest}, which diff --git a/java/core/src/test/java/com/google/protobuf/ExtensionRegistryFactoryTest.java b/java/core/src/test/java/com/google/protobuf/ExtensionRegistryFactoryTest.java index c1246782..6157e589 100644 --- a/java/core/src/test/java/com/google/protobuf/ExtensionRegistryFactoryTest.java +++ b/java/core/src/test/java/com/google/protobuf/ExtensionRegistryFactoryTest.java @@ -32,26 +32,24 @@ package com.google.protobuf; import protobuf_unittest.NonNestedExtension; import protobuf_unittest.NonNestedExtensionLite; - -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - import java.lang.reflect.Method; import java.net.URLClassLoader; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.Set; +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; /** * Tests for {@link ExtensionRegistryFactory} and the {@link ExtensionRegistry} instances it * creates. - * + * * <p>This test simulates the runtime behaviour of the ExtensionRegistryFactory by delegating test * definitions to two inner classes {@link InnerTest} and {@link InnerLiteTest}, the latter of * which is executed using a custom ClassLoader, simulating the ProtoLite environment. - * + * * <p>The test mechanism employed here is based on the pattern in * {@code com.google.common.util.concurrent.AbstractFutureFallbackAtomicHelperTest} */ @@ -68,6 +66,7 @@ public class ExtensionRegistryFactoryTest extends TestCase { void testEmpty(); void testIsFullRegistry(); void testAdd(); + void testAdd_immutable(); } /** @@ -125,6 +124,29 @@ public class ExtensionRegistryFactoryTest extends TestCase { assertNotNull("Extension is registered in masqueraded full registry", fullRegistry2.findImmutableExtensionByName("protobuf_unittest.nonNestedExtension")); } + + @Override + public void testAdd_immutable() { + ExtensionRegistryLite registry1 = ExtensionRegistryLite.newInstance().getUnmodifiable(); + try { + NonNestedExtensionLite.registerAllExtensions(registry1); + fail(); + } catch (UnsupportedOperationException expected) {} + try { + registry1.add(NonNestedExtensionLite.nonNestedExtensionLite); + fail(); + } catch (UnsupportedOperationException expected) {} + + ExtensionRegistryLite registry2 = ExtensionRegistryLite.newInstance().getUnmodifiable(); + try { + NonNestedExtension.registerAllExtensions((ExtensionRegistry) registry2); + fail(); + } catch (IllegalArgumentException expected) {} + try { + registry2.add(NonNestedExtension.nonNestedExtension); + fail(); + } catch (IllegalArgumentException expected) {} + } } /** @@ -162,12 +184,21 @@ public class ExtensionRegistryFactoryTest extends TestCase { NonNestedExtensionLite.MessageLiteToBeExtended.getDefaultInstance(), 1); assertNotNull("Extension is registered in Lite registry", extension); } + + @Override + public void testAdd_immutable() { + ExtensionRegistryLite registry = ExtensionRegistryLite.newInstance().getUnmodifiable(); + try { + NonNestedExtensionLite.registerAllExtensions(registry); + fail(); + } catch (UnsupportedOperationException expected) {} + } } /** * Defines a suite of tests which the JUnit3 runner retrieves by reflection. */ - public static Test suite() { + public static Test suite() { TestSuite suite = new TestSuite(); for (Method method : RegistryTests.class.getMethods()) { suite.addTest(TestSuite.createTest(ExtensionRegistryFactoryTest.class, method.getName())); diff --git a/java/core/src/test/java/com/google/protobuf/FloatArrayListTest.java b/java/core/src/test/java/com/google/protobuf/FloatArrayListTest.java index 0e13a598..aa36be49 100644 --- a/java/core/src/test/java/com/google/protobuf/FloatArrayListTest.java +++ b/java/core/src/test/java/com/google/protobuf/FloatArrayListTest.java @@ -32,11 +32,10 @@ package com.google.protobuf; import static java.util.Arrays.asList; -import junit.framework.TestCase; - import java.util.Collections; import java.util.ConcurrentModificationException; import java.util.Iterator; +import junit.framework.TestCase; /** * Tests for {@link FloatArrayList}. diff --git a/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java b/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java index 127e06fd..3eece26a 100644 --- a/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java +++ b/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java @@ -65,9 +65,6 @@ import protobuf_unittest.UnittestProto.TestExtremeDefaultValues; import protobuf_unittest.UnittestProto.TestOneof2; import protobuf_unittest.UnittestProto.TestPackedTypes; import protobuf_unittest.UnittestProto.TestUnpackedTypes; - -import junit.framework.TestCase; - import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.ObjectInputStream; @@ -76,6 +73,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; +import junit.framework.TestCase; /** * Unit test for generated messages and generated code. See also diff --git a/java/core/src/test/java/com/google/protobuf/IntArrayListTest.java b/java/core/src/test/java/com/google/protobuf/IntArrayListTest.java index e59e3c6e..60c85450 100644 --- a/java/core/src/test/java/com/google/protobuf/IntArrayListTest.java +++ b/java/core/src/test/java/com/google/protobuf/IntArrayListTest.java @@ -32,11 +32,10 @@ package com.google.protobuf; import static java.util.Arrays.asList; -import junit.framework.TestCase; - import java.util.Collections; import java.util.ConcurrentModificationException; import java.util.Iterator; +import junit.framework.TestCase; /** * Tests for {@link IntArrayList}. diff --git a/java/core/src/test/java/com/google/protobuf/LazyFieldLiteTest.java b/java/core/src/test/java/com/google/protobuf/LazyFieldLiteTest.java index 8d1de6dc..efdfdfbe 100644 --- a/java/core/src/test/java/com/google/protobuf/LazyFieldLiteTest.java +++ b/java/core/src/test/java/com/google/protobuf/LazyFieldLiteTest.java @@ -35,10 +35,8 @@ import static protobuf_unittest.UnittestProto.optionalInt64Extension; import protobuf_unittest.UnittestProto.TestAllExtensions; import protobuf_unittest.UnittestProto.TestAllTypes; - -import junit.framework.TestCase; - import java.io.IOException; +import junit.framework.TestCase; /** * Unit test for {@link LazyFieldLite}. diff --git a/java/core/src/test/java/com/google/protobuf/LazyFieldTest.java b/java/core/src/test/java/com/google/protobuf/LazyFieldTest.java index ce473ae9..5f013f3c 100644 --- a/java/core/src/test/java/com/google/protobuf/LazyFieldTest.java +++ b/java/core/src/test/java/com/google/protobuf/LazyFieldTest.java @@ -32,10 +32,8 @@ package com.google.protobuf; import protobuf_unittest.UnittestProto.TestAllExtensions; import protobuf_unittest.UnittestProto.TestAllTypes; - -import junit.framework.TestCase; - import java.io.IOException; +import junit.framework.TestCase; /** * Unit test for {@link LazyField}. diff --git a/java/core/src/test/java/com/google/protobuf/LazyMessageLiteTest.java b/java/core/src/test/java/com/google/protobuf/LazyMessageLiteTest.java index e5b11cf1..c8ca9b6a 100644 --- a/java/core/src/test/java/com/google/protobuf/LazyMessageLiteTest.java +++ b/java/core/src/test/java/com/google/protobuf/LazyMessageLiteTest.java @@ -34,10 +34,8 @@ import protobuf_unittest.LazyFieldsLite.LazyExtension; import protobuf_unittest.LazyFieldsLite.LazyInnerMessageLite; import protobuf_unittest.LazyFieldsLite.LazyMessageLite; import protobuf_unittest.LazyFieldsLite.LazyNestedInnerMessageLite; - -import junit.framework.TestCase; - import java.util.ArrayList; +import junit.framework.TestCase; /** * Unit test for messages with lazy fields. diff --git a/java/core/src/test/java/com/google/protobuf/LazyStringArrayListTest.java b/java/core/src/test/java/com/google/protobuf/LazyStringArrayListTest.java index 497c4df2..d2bee2a4 100644 --- a/java/core/src/test/java/com/google/protobuf/LazyStringArrayListTest.java +++ b/java/core/src/test/java/com/google/protobuf/LazyStringArrayListTest.java @@ -32,13 +32,12 @@ package com.google.protobuf; import static java.util.Arrays.asList; -import junit.framework.TestCase; - import java.util.ArrayList; import java.util.Collections; import java.util.ConcurrentModificationException; import java.util.Iterator; import java.util.List; +import junit.framework.TestCase; /** * Tests for {@link LazyStringArrayList}. diff --git a/java/core/src/test/java/com/google/protobuf/LazyStringEndToEndTest.java b/java/core/src/test/java/com/google/protobuf/LazyStringEndToEndTest.java index 0ef414aa..006e4933 100644 --- a/java/core/src/test/java/com/google/protobuf/LazyStringEndToEndTest.java +++ b/java/core/src/test/java/com/google/protobuf/LazyStringEndToEndTest.java @@ -32,10 +32,8 @@ package com.google.protobuf; import protobuf_unittest.UnittestProto; - -import junit.framework.TestCase; - import java.io.IOException; +import junit.framework.TestCase; /** * Tests to make sure the lazy conversion of UTF8-encoded byte arrays to diff --git a/java/core/src/test/java/com/google/protobuf/LiteTest.java b/java/core/src/test/java/com/google/protobuf/LiteTest.java index b3a246dc..254beba1 100644 --- a/java/core/src/test/java/com/google/protobuf/LiteTest.java +++ b/java/core/src/test/java/com/google/protobuf/LiteTest.java @@ -35,7 +35,6 @@ import static java.util.Collections.singletonList; import com.google.protobuf.UnittestImportLite.ImportEnumLite; import com.google.protobuf.UnittestImportPublicLite.PublicImportMessageLite; -import com.google.protobuf.UnittestLite; import com.google.protobuf.UnittestLite.ForeignEnumLite; import com.google.protobuf.UnittestLite.ForeignMessageLite; import com.google.protobuf.UnittestLite.TestAllExtensionsLite; @@ -52,15 +51,8 @@ import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.BarPrime; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.Foo; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.TestOneofEquals; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.TestRecursiveOneof; - import junit.framework.TestCase; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.NotSerializableException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; - /** * Test lite runtime. * @@ -139,7 +131,6 @@ public class LiteTest extends TestCase { UnittestLite.optionalNestedMessageExtensionLite).getBb()); } - public void testClone() { TestAllTypesLite.Builder expected = TestAllTypesLite.newBuilder() .setOptionalInt32(123); diff --git a/java/core/src/test/java/com/google/protobuf/LiteralByteStringTest.java b/java/core/src/test/java/com/google/protobuf/LiteralByteStringTest.java index 2e7792a8..eac47448 100644 --- a/java/core/src/test/java/com/google/protobuf/LiteralByteStringTest.java +++ b/java/core/src/test/java/com/google/protobuf/LiteralByteStringTest.java @@ -30,8 +30,6 @@ package com.google.protobuf; -import junit.framework.TestCase; - import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.EOFException; @@ -45,6 +43,7 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; import java.util.NoSuchElementException; +import junit.framework.TestCase; /** * Test {@code LiteralByteString} by setting up a reference string in {@link #setUp()}. diff --git a/java/core/src/test/java/com/google/protobuf/LongArrayListTest.java b/java/core/src/test/java/com/google/protobuf/LongArrayListTest.java index 6aaf85d7..6bbdfcaa 100644 --- a/java/core/src/test/java/com/google/protobuf/LongArrayListTest.java +++ b/java/core/src/test/java/com/google/protobuf/LongArrayListTest.java @@ -32,11 +32,10 @@ package com.google.protobuf; import static java.util.Arrays.asList; -import junit.framework.TestCase; - import java.util.Collections; import java.util.ConcurrentModificationException; import java.util.Iterator; +import junit.framework.TestCase; /** * Tests for {@link LongArrayList}. diff --git a/java/core/src/test/java/com/google/protobuf/MapForProto2LiteTest.java b/java/core/src/test/java/com/google/protobuf/MapForProto2LiteTest.java index 04d58006..8b64fa27 100644 --- a/java/core/src/test/java/com/google/protobuf/MapForProto2LiteTest.java +++ b/java/core/src/test/java/com/google/protobuf/MapForProto2LiteTest.java @@ -35,15 +35,13 @@ import map_lite_test.MapForProto2TestProto.TestMap; import map_lite_test.MapForProto2TestProto.TestMap.MessageValue; import map_lite_test.MapForProto2TestProto.TestMapOrBuilder; import map_lite_test.MapForProto2TestProto.TestUnknownEnumValue; - -import junit.framework.TestCase; - import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import junit.framework.TestCase; /** * Unit tests for map fields. diff --git a/java/core/src/test/java/com/google/protobuf/MapForProto2Test.java b/java/core/src/test/java/com/google/protobuf/MapForProto2Test.java index e8246bf6..76bd7bd9 100644 --- a/java/core/src/test/java/com/google/protobuf/MapForProto2Test.java +++ b/java/core/src/test/java/com/google/protobuf/MapForProto2Test.java @@ -32,14 +32,14 @@ package com.google.protobuf; import com.google.protobuf.Descriptors.FieldDescriptor; import map_test.MapForProto2TestProto.BizarroTestMap; +import map_test.MapForProto2TestProto.ReservedAsMapField; +import map_test.MapForProto2TestProto.ReservedAsMapFieldWithEnumValue; import map_test.MapForProto2TestProto.TestMap; import map_test.MapForProto2TestProto.TestMap.MessageValue; import map_test.MapForProto2TestProto.TestMap.MessageWithRequiredFields; import map_test.MapForProto2TestProto.TestMapOrBuilder; import map_test.MapForProto2TestProto.TestRecursiveMap; import map_test.MapForProto2TestProto.TestUnknownEnumValue; -import junit.framework.TestCase; - import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.ArrayList; @@ -47,6 +47,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import junit.framework.TestCase; /** * Unit tests for map fields in proto2 protos. @@ -1143,6 +1144,11 @@ public class MapForProto2Test extends TestCase { assertEquals(2, mapEntry.getAllFields().size()); } + public void testReservedWordsFieldNames() { + ReservedAsMapField.newBuilder().build(); + ReservedAsMapFieldWithEnumValue.newBuilder().build(); + } + private static <K, V> Map<K, V> newMap(K key1, V value1) { Map<K, V> map = new HashMap<K, V>(); map.put(key1, value1); diff --git a/java/core/src/test/java/com/google/protobuf/MapTest.java b/java/core/src/test/java/com/google/protobuf/MapTest.java index caef246b..035a8335 100644 --- a/java/core/src/test/java/com/google/protobuf/MapTest.java +++ b/java/core/src/test/java/com/google/protobuf/MapTest.java @@ -36,12 +36,12 @@ import com.google.protobuf.Descriptors.EnumDescriptor; import com.google.protobuf.Descriptors.EnumValueDescriptor; import com.google.protobuf.Descriptors.FieldDescriptor; import map_test.MapTestProto.BizarroTestMap; +import map_test.MapTestProto.ReservedAsMapField; +import map_test.MapTestProto.ReservedAsMapFieldWithEnumValue; import map_test.MapTestProto.TestMap; import map_test.MapTestProto.TestMap.MessageValue; import map_test.MapTestProto.TestMapOrBuilder; import map_test.MapTestProto.TestOnChangeEventPropagation; -import junit.framework.TestCase; - import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.ArrayList; @@ -49,6 +49,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import junit.framework.TestCase; /** * Unit tests for map fields. @@ -1307,6 +1308,171 @@ public class MapTest extends TestCase { } } + public void testReservedWordsFieldNames() { + ReservedAsMapField.newBuilder().build(); + ReservedAsMapFieldWithEnumValue.newBuilder().build(); + } + + public void testDeterministicSerialziation() throws Exception { + TestMap.Builder builder = TestMap.newBuilder(); + // int32->int32 + builder.putInt32ToInt32Field(5, 1); + builder.putInt32ToInt32Field(1, 1); + builder.putInt32ToInt32Field(4, 1); + builder.putInt32ToInt32Field(-2, 1); + builder.putInt32ToInt32Field(0, 1); + + // uint32->int32 + builder.putUint32ToInt32Field(5, 1); + builder.putUint32ToInt32Field(1, 1); + builder.putUint32ToInt32Field(4, 1); + builder.putUint32ToInt32Field(-2, 1); + builder.putUint32ToInt32Field(0, 1); + + // int64->int32 + builder.putInt64ToInt32Field(5L, 1); + builder.putInt64ToInt32Field(1L, 1); + builder.putInt64ToInt32Field(4L, 1); + builder.putInt64ToInt32Field(-2L, 1); + builder.putInt64ToInt32Field(0L, 1); + + // string->int32 + builder.putStringToInt32Field("baz", 1); + builder.putStringToInt32Field("foo", 1); + builder.putStringToInt32Field("bar", 1); + builder.putStringToInt32Field("", 1); + builder.putStringToInt32Field("hello", 1); + builder.putStringToInt32Field("world", 1); + + TestMap message = builder.build(); + byte[] serialized = new byte[message.getSerializedSize()]; + CodedOutputStream output = CodedOutputStream.newInstance(serialized); + output.useDeterministicSerialization(); + message.writeTo(output); + output.flush(); + + CodedInputStream input = CodedInputStream.newInstance(serialized); + List<Integer> int32Keys = new ArrayList<Integer>(); + List<Integer> uint32Keys = new ArrayList<Integer>(); + List<Long> int64Keys = new ArrayList<Long>(); + List<String> stringKeys = new ArrayList<String>(); + int tag; + while (true) { + tag = input.readTag(); + if (tag == 0) { + break; + } + int length = input.readRawVarint32(); + int oldLimit = input.pushLimit(length); + switch (WireFormat.getTagFieldNumber(tag)) { + case TestMap.STRING_TO_INT32_FIELD_FIELD_NUMBER: + stringKeys.add(readMapStringKey(input)); + break; + case TestMap.INT32_TO_INT32_FIELD_FIELD_NUMBER: + int32Keys.add(readMapIntegerKey(input)); + break; + case TestMap.UINT32_TO_INT32_FIELD_FIELD_NUMBER: + uint32Keys.add(readMapIntegerKey(input)); + break; + case TestMap.INT64_TO_INT32_FIELD_FIELD_NUMBER: + int64Keys.add(readMapLongKey(input)); + break; + default: + fail("Unexpected fields."); + } + input.popLimit(oldLimit); + } + assertEquals( + Arrays.asList(-2, 0, 1, 4, 5), + int32Keys); + assertEquals( + Arrays.asList(-2, 0, 1, 4, 5), + uint32Keys); + assertEquals( + Arrays.asList(-2L, 0L, 1L, 4L, 5L), + int64Keys); + assertEquals( + Arrays.asList("", "bar", "baz", "foo", "hello", "world"), + stringKeys); + } + + public void testInitFromPartialDynamicMessage() { + FieldDescriptor fieldDescriptor = + TestMap.getDescriptor().findFieldByNumber(TestMap.INT32_TO_MESSAGE_FIELD_FIELD_NUMBER); + Descriptor mapEntryType = fieldDescriptor.getMessageType(); + FieldDescriptor keyField = mapEntryType.findFieldByNumber(1); + FieldDescriptor valueField = mapEntryType.findFieldByNumber(2); + DynamicMessage dynamicMessage = + DynamicMessage.newBuilder(TestMap.getDescriptor()) + .addRepeatedField( + fieldDescriptor, + DynamicMessage.newBuilder(mapEntryType) + .setField(keyField, 10) + .setField(valueField, TestMap.MessageValue.newBuilder().setValue(10).build()) + .build()) + .build(); + TestMap message = TestMap.newBuilder().mergeFrom(dynamicMessage).build(); + assertEquals( + TestMap.MessageValue.newBuilder().setValue(10).build(), + message.getInt32ToMessageFieldMap().get(10)); + } + + public void testInitFromFullyDynamicMessage() { + FieldDescriptor fieldDescriptor = + TestMap.getDescriptor().findFieldByNumber(TestMap.INT32_TO_MESSAGE_FIELD_FIELD_NUMBER); + Descriptor mapEntryType = fieldDescriptor.getMessageType(); + FieldDescriptor keyField = mapEntryType.findFieldByNumber(1); + FieldDescriptor valueField = mapEntryType.findFieldByNumber(2); + DynamicMessage dynamicMessage = + DynamicMessage.newBuilder(TestMap.getDescriptor()) + .addRepeatedField( + fieldDescriptor, + DynamicMessage.newBuilder(mapEntryType) + .setField(keyField, 10) + .setField( + valueField, + DynamicMessage.newBuilder(TestMap.MessageValue.getDescriptor()) + .setField( + TestMap.MessageValue.getDescriptor().findFieldByName("value"), 10) + .build()) + .build()) + .build(); + TestMap message = TestMap.newBuilder().mergeFrom(dynamicMessage).build(); + assertEquals( + TestMap.MessageValue.newBuilder().setValue(10).build(), + message.getInt32ToMessageFieldMap().get(10)); + } + + private int readMapIntegerKey(CodedInputStream input) throws IOException { + int tag = input.readTag(); + assertEquals(WireFormat.makeTag(1, WireFormat.WIRETYPE_VARINT), tag); + int ret = input.readInt32(); + // skip the value field. + input.skipField(input.readTag()); + assertTrue(input.isAtEnd()); + return ret; + } + + private long readMapLongKey(CodedInputStream input) throws IOException { + int tag = input.readTag(); + assertEquals(WireFormat.makeTag(1, WireFormat.WIRETYPE_VARINT), tag); + long ret = input.readInt64(); + // skip the value field. + input.skipField(input.readTag()); + assertTrue(input.isAtEnd()); + return ret; + } + + private String readMapStringKey(CodedInputStream input) throws IOException { + int tag = input.readTag(); + assertEquals(WireFormat.makeTag(1, WireFormat.WIRETYPE_LENGTH_DELIMITED), tag); + String ret = input.readString(); + // skip the value field. + input.skipField(input.readTag()); + assertTrue(input.isAtEnd()); + return ret; + } + private static <K, V> Map<K, V> newMap(K key1, V value1) { Map<K, V> map = new HashMap<K, V>(); map.put(key1, value1); diff --git a/java/core/src/test/java/com/google/protobuf/MessageTest.java b/java/core/src/test/java/com/google/protobuf/MessageTest.java index dcd1aba7..75b79a34 100644 --- a/java/core/src/test/java/com/google/protobuf/MessageTest.java +++ b/java/core/src/test/java/com/google/protobuf/MessageTest.java @@ -35,10 +35,8 @@ import protobuf_unittest.UnittestProto.TestAllExtensions; import protobuf_unittest.UnittestProto.TestAllTypes; import protobuf_unittest.UnittestProto.TestRequired; import protobuf_unittest.UnittestProto.TestRequiredForeign; - -import junit.framework.TestCase; - import java.util.List; +import junit.framework.TestCase; /** * Misc. unit tests for message operations that apply to both generated diff --git a/java/core/src/test/java/com/google/protobuf/NestedBuildersTest.java b/java/core/src/test/java/com/google/protobuf/NestedBuildersTest.java index 542e28c0..03ed65a5 100644 --- a/java/core/src/test/java/com/google/protobuf/NestedBuildersTest.java +++ b/java/core/src/test/java/com/google/protobuf/NestedBuildersTest.java @@ -32,11 +32,9 @@ package com.google.protobuf; import protobuf_unittest.Vehicle; import protobuf_unittest.Wheel; - -import junit.framework.TestCase; - import java.util.ArrayList; import java.util.List; +import junit.framework.TestCase; /** * Test cases that exercise end-to-end use cases involving diff --git a/java/core/src/test/java/com/google/protobuf/NioByteStringTest.java b/java/core/src/test/java/com/google/protobuf/NioByteStringTest.java index 6be5b93c..c388bd05 100644 --- a/java/core/src/test/java/com/google/protobuf/NioByteStringTest.java +++ b/java/core/src/test/java/com/google/protobuf/NioByteStringTest.java @@ -32,8 +32,6 @@ package com.google.protobuf; import static com.google.protobuf.Internal.UTF_8; -import junit.framework.TestCase; - import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.EOFException; @@ -48,6 +46,7 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; import java.util.NoSuchElementException; +import junit.framework.TestCase; /** * Tests for {@link NioByteString}. diff --git a/java/core/src/test/java/com/google/protobuf/ParseExceptionsTest.java b/java/core/src/test/java/com/google/protobuf/ParseExceptionsTest.java index bf1f1d71..e376b1cd 100644 --- a/java/core/src/test/java/com/google/protobuf/ParseExceptionsTest.java +++ b/java/core/src/test/java/com/google/protobuf/ParseExceptionsTest.java @@ -36,16 +36,14 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import com.google.protobuf.DescriptorProtos.DescriptorProto; - -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; /** * Tests the exceptions thrown when parsing from a stream. The methods on the {@link Parser} diff --git a/java/core/src/test/java/com/google/protobuf/ParserTest.java b/java/core/src/test/java/com/google/protobuf/ParserTest.java index 30842d2c..7086912b 100644 --- a/java/core/src/test/java/com/google/protobuf/ParserTest.java +++ b/java/core/src/test/java/com/google/protobuf/ParserTest.java @@ -42,13 +42,11 @@ import protobuf_unittest.UnittestProto.TestAllTypes; import protobuf_unittest.UnittestProto.TestEmptyMessage; import protobuf_unittest.UnittestProto.TestParsingMerge; import protobuf_unittest.UnittestProto.TestRequired; - -import junit.framework.TestCase; - import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import junit.framework.TestCase; /** * Unit test for {@link Parser}. diff --git a/java/core/src/test/java/com/google/protobuf/ProtobufArrayListTest.java b/java/core/src/test/java/com/google/protobuf/ProtobufArrayListTest.java index 3f45e226..af717bfd 100644 --- a/java/core/src/test/java/com/google/protobuf/ProtobufArrayListTest.java +++ b/java/core/src/test/java/com/google/protobuf/ProtobufArrayListTest.java @@ -32,12 +32,11 @@ package com.google.protobuf; import static java.util.Arrays.asList; -import junit.framework.TestCase; - import java.util.Collections; import java.util.ConcurrentModificationException; import java.util.Iterator; import java.util.List; +import junit.framework.TestCase; /** * Tests for {@link ProtobufArrayList}. diff --git a/java/core/src/test/java/com/google/protobuf/RepeatedFieldBuilderV3Test.java b/java/core/src/test/java/com/google/protobuf/RepeatedFieldBuilderV3Test.java index 241a4354..edbd0afd 100644 --- a/java/core/src/test/java/com/google/protobuf/RepeatedFieldBuilderV3Test.java +++ b/java/core/src/test/java/com/google/protobuf/RepeatedFieldBuilderV3Test.java @@ -32,11 +32,9 @@ package com.google.protobuf; import protobuf_unittest.UnittestProto.TestAllTypes; import protobuf_unittest.UnittestProto.TestAllTypesOrBuilder; - -import junit.framework.TestCase; - import java.util.Collections; import java.util.List; +import junit.framework.TestCase; /** * Tests for {@link RepeatedFieldBuilderV3}. This tests basic functionality. diff --git a/java/core/src/test/java/com/google/protobuf/ServiceTest.java b/java/core/src/test/java/com/google/protobuf/ServiceTest.java index b902737d..b895ad8d 100644 --- a/java/core/src/test/java/com/google/protobuf/ServiceTest.java +++ b/java/core/src/test/java/com/google/protobuf/ServiceTest.java @@ -42,15 +42,13 @@ import protobuf_unittest.UnittestProto.FooResponse; import protobuf_unittest.UnittestProto.TestAllTypes; import protobuf_unittest.UnittestProto.TestService; +import java.util.HashSet; +import java.util.Set; import junit.framework.TestCase; - import org.easymock.classextension.EasyMock; import org.easymock.IArgumentMatcher; import org.easymock.classextension.IMocksControl; -import java.util.HashSet; -import java.util.Set; - /** * Tests services and stubs. * @@ -271,6 +269,8 @@ public class ServiceTest extends TestCase { file.getServices().get(0).getMethods().get(0).getName()); } + + // ================================================================= /** diff --git a/java/core/src/test/java/com/google/protobuf/SmallSortedMapTest.java b/java/core/src/test/java/com/google/protobuf/SmallSortedMapTest.java index e96ecd65..a7f8342d 100644 --- a/java/core/src/test/java/com/google/protobuf/SmallSortedMapTest.java +++ b/java/core/src/test/java/com/google/protobuf/SmallSortedMapTest.java @@ -30,8 +30,6 @@ package com.google.protobuf; -import junit.framework.TestCase; - import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -40,6 +38,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; +import junit.framework.TestCase; /** * @author darick@google.com Darick Tong diff --git a/java/core/src/test/java/com/google/protobuf/TestUtil.java b/java/core/src/test/java/com/google/protobuf/TestUtil.java index d94f99e3..d4a18a22 100644 --- a/java/core/src/test/java/com/google/protobuf/TestUtil.java +++ b/java/core/src/test/java/com/google/protobuf/TestUtil.java @@ -232,12 +232,10 @@ import protobuf_unittest.UnittestProto.TestOneof2; import protobuf_unittest.UnittestProto.TestPackedExtensions; import protobuf_unittest.UnittestProto.TestPackedTypes; import protobuf_unittest.UnittestProto.TestUnpackedTypes; - -import junit.framework.Assert; - import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; +import junit.framework.Assert; /** * Contains methods for setting all fields of {@code TestAllTypes} to @@ -260,6 +258,13 @@ public final class TestUtil { } /** + * Dirties the message by resetting the momoized serialized size. + */ + public static void resetMemoizedSize(AbstractMessage message) { + message.memoizedSize = -1; + } + + /** * Get a {@code TestAllTypes} with all fields set as they would be by * {@link #setAllFields(TestAllTypes.Builder)}. */ diff --git a/java/core/src/test/java/com/google/protobuf/TextFormatTest.java b/java/core/src/test/java/com/google/protobuf/TextFormatTest.java index 14783b0a..6a91b02f 100644 --- a/java/core/src/test/java/com/google/protobuf/TextFormatTest.java +++ b/java/core/src/test/java/com/google/protobuf/TextFormatTest.java @@ -42,11 +42,9 @@ import protobuf_unittest.UnittestProto.TestAllTypes.NestedMessage; import protobuf_unittest.UnittestProto.TestEmptyMessage; import protobuf_unittest.UnittestProto.TestOneof2; import proto2_wireformat_unittest.UnittestMsetWireFormat.TestMessageSet; - -import junit.framework.TestCase; - import java.io.StringReader; import java.util.List; +import junit.framework.TestCase; /** * Test case for {@link TextFormat}. @@ -992,10 +990,35 @@ public class TextFormatTest extends TestCase { assertParseSuccessWithOverwriteForbidden("repeated_nested_message [{ bb: 1 }, { bb: 2 }]\n"); } + public void testParseShortRepeatedFormOfEmptyRepeatedFields() throws Exception { + assertParseSuccessWithOverwriteForbidden("repeated_foreign_enum: []"); + assertParseSuccessWithOverwriteForbidden("repeated_int32: []\n"); + assertParseSuccessWithOverwriteForbidden("RepeatedGroup []\n"); + assertParseSuccessWithOverwriteForbidden("repeated_nested_message []\n"); + } + + public void testParseShortRepeatedFormWithTrailingComma() throws Exception { + assertParseErrorWithOverwriteForbidden( + "1:38: Expected identifier. Found \']\'", + "repeated_foreign_enum: [FOREIGN_FOO, ]\n"); + assertParseErrorWithOverwriteForbidden( + "1:22: Couldn't parse integer: For input string: \"]\"", + "repeated_int32: [ 1, ]\n"); + assertParseErrorWithOverwriteForbidden( + "1:25: Expected \"{\".", + "RepeatedGroup [{ a: 1 },]\n"); + assertParseErrorWithOverwriteForbidden( + "1:37: Expected \"{\".", + "repeated_nested_message [{ bb: 1 }, ]\n"); + } + public void testParseShortRepeatedFormOfNonRepeatedFields() throws Exception { assertParseErrorWithOverwriteForbidden( "1:17: Couldn't parse integer: For input string: \"[\"", "optional_int32: [1]\n"); + assertParseErrorWithOverwriteForbidden( + "1:17: Couldn't parse integer: For input string: \"[\"", + "optional_int32: []\n"); } // ======================================================================= diff --git a/java/core/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java b/java/core/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java index 573cd665..26ea850e 100644 --- a/java/core/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java +++ b/java/core/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java @@ -35,11 +35,9 @@ import com.google.protobuf.UnittestLite.TestAllTypesLite; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.Bar; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.Foo; - -import junit.framework.TestCase; - import java.io.ByteArrayOutputStream; import java.io.IOException; +import junit.framework.TestCase; /** * Tests for {@link UnknownFieldSetLite}. diff --git a/java/core/src/test/java/com/google/protobuf/UnknownFieldSetTest.java b/java/core/src/test/java/com/google/protobuf/UnknownFieldSetTest.java index 32380f70..f81e90b4 100644 --- a/java/core/src/test/java/com/google/protobuf/UnknownFieldSetTest.java +++ b/java/core/src/test/java/com/google/protobuf/UnknownFieldSetTest.java @@ -38,11 +38,9 @@ import protobuf_unittest.UnittestProto.TestEmptyMessage; import protobuf_unittest.UnittestProto.TestEmptyMessageWithExtensions; import protobuf_unittest.UnittestProto.TestPackedExtensions; import protobuf_unittest.UnittestProto.TestPackedTypes; - -import junit.framework.TestCase; - import java.util.Arrays; import java.util.Map; +import junit.framework.TestCase; /** * Tests related to unknown field handling. diff --git a/java/core/src/test/java/com/google/protobuf/UnmodifiableLazyStringListTest.java b/java/core/src/test/java/com/google/protobuf/UnmodifiableLazyStringListTest.java index b1c75fc3..00f201ca 100644 --- a/java/core/src/test/java/com/google/protobuf/UnmodifiableLazyStringListTest.java +++ b/java/core/src/test/java/com/google/protobuf/UnmodifiableLazyStringListTest.java @@ -30,11 +30,10 @@ package com.google.protobuf; -import junit.framework.TestCase; - import java.util.Iterator; import java.util.List; import java.util.ListIterator; +import junit.framework.TestCase; /** * Tests for {@link UnmodifiableLazyStringList}. diff --git a/java/core/src/test/java/com/google/protobuf/WireFormatTest.java b/java/core/src/test/java/com/google/protobuf/WireFormatTest.java index e66b371c..370860c2 100644 --- a/java/core/src/test/java/com/google/protobuf/WireFormatTest.java +++ b/java/core/src/test/java/com/google/protobuf/WireFormatTest.java @@ -44,12 +44,10 @@ import protobuf_unittest.UnittestProto.TestOneofBackwardsCompatible; import protobuf_unittest.UnittestProto.TestPackedExtensions; import protobuf_unittest.UnittestProto.TestPackedTypes; import proto2_wireformat_unittest.UnittestMsetWireFormat.TestMessageSet; - -import junit.framework.TestCase; - import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.util.List; +import junit.framework.TestCase; /** * Tests related to parsing and serialization. diff --git a/java/core/src/test/proto/com/google/protobuf/field_presence_test.proto b/java/core/src/test/proto/com/google/protobuf/field_presence_test.proto index 86cdd286..2367bd8b 100644 --- a/java/core/src/test/proto/com/google/protobuf/field_presence_test.proto +++ b/java/core/src/test/proto/com/google/protobuf/field_presence_test.proto @@ -36,7 +36,6 @@ import "google/protobuf/unittest.proto"; option java_package = "com.google.protobuf"; option java_outer_classname = "FieldPresenceTestProto"; -option java_generate_equals_and_hash = true; message TestAllTypes { enum NestedEnum { diff --git a/java/core/src/test/proto/com/google/protobuf/map_for_proto2_lite_test.proto b/java/core/src/test/proto/com/google/protobuf/map_for_proto2_lite_test.proto index de3336a5..2ca0251c 100644 --- a/java/core/src/test/proto/com/google/protobuf/map_for_proto2_lite_test.proto +++ b/java/core/src/test/proto/com/google/protobuf/map_for_proto2_lite_test.proto @@ -32,7 +32,6 @@ syntax = "proto2"; option java_outer_classname = "MapForProto2TestProto"; -option java_generate_equals_and_hash = true; message TestMap { message MessageValue { @@ -81,6 +80,42 @@ message BizarroTestMap { map<string, bytes> int32_to_message_field = 5; // different key and value types map<string, bytes> string_to_int32_field = 6; // same key type, different value } + +// Used to test that java reserved words can be used as protobuf field names +// Not all reserved words are tested (to avoid bloat) but instead an arbitrary +// subset of them chosen to cover various keyword categories like +// type, modifier, declaration, etc. +message ReservedAsMapField { + map<string, uint32> if = 1; + map<string, uint32> const = 2; + map<string, uint32> private = 3; + map<string, uint32> class = 4; + map<string, uint32> int = 5; + map<string, uint32> void = 6; + map<string, uint32> string = 7; // These are also proto keywords + map<string, uint32> package = 8; + map<string, uint32> enum = 9; // Most recent Java reserved word + map<string, uint32> null = 10; + // null is not a 'reserved word' per se but as a literal needs similar care +} + +message ReservedAsMapFieldWithEnumValue { + enum SampleEnum { + A = 0; + B = 1; + } + map<string, SampleEnum> if = 1; + map<string, SampleEnum> const = 2; + map<string, SampleEnum> private = 3; + map<string, SampleEnum> class = 4; + map<string, SampleEnum> int = 5; + map<string, SampleEnum> void = 6; + map<string, SampleEnum> string = 7; // These are also proto keywords + map<string, SampleEnum> package = 8; + map<string, SampleEnum> enum = 9; // Most recent Java reserved word + map<string, SampleEnum> null = 10; + // null is not a 'reserved word' per se but as a literal needs similar care +} package map_for_proto2_lite_test; option java_package = "map_lite_test"; option optimize_for = LITE_RUNTIME; diff --git a/java/core/src/test/proto/com/google/protobuf/map_for_proto2_test.proto b/java/core/src/test/proto/com/google/protobuf/map_for_proto2_test.proto index 0c92b0ae..974f8a2c 100644 --- a/java/core/src/test/proto/com/google/protobuf/map_for_proto2_test.proto +++ b/java/core/src/test/proto/com/google/protobuf/map_for_proto2_test.proto @@ -34,7 +34,6 @@ package map_for_proto2_test; option java_package = "map_test"; option java_outer_classname = "MapForProto2TestProto"; -option java_generate_equals_and_hash = true; message TestMap { message MessageValue { @@ -83,3 +82,39 @@ message BizarroTestMap { map<string, bytes> int32_to_message_field = 5; // different key and value types map<string, bytes> string_to_int32_field = 6; // same key type, different value } + +// Used to test that java reserved words can be used as protobuf field names +// Not all reserved words are tested (to avoid bloat) but instead an arbitrary +// subset of them chosen to cover various keyword categories like +// type, modifier, declaration, etc. +message ReservedAsMapField { + map<string, uint32> if = 1; + map<string, uint32> const = 2; + map<string, uint32> private = 3; + map<string, uint32> class = 4; + map<string, uint32> int = 5; + map<string, uint32> void = 6; + map<string, uint32> string = 7; // These are also proto keywords + map<string, uint32> package = 8; + map<string, uint32> enum = 9; // Most recent Java reserved word + map<string, uint32> null = 10; + // null is not a 'reserved word' per se but as a literal needs similar care +} + +message ReservedAsMapFieldWithEnumValue { + enum SampleEnum { + A = 0; + B = 1; + } + map<string, SampleEnum> if = 1; + map<string, SampleEnum> const = 2; + map<string, SampleEnum> private = 3; + map<string, SampleEnum> class = 4; + map<string, SampleEnum> int = 5; + map<string, SampleEnum> void = 6; + map<string, SampleEnum> string = 7; // These are also proto keywords + map<string, SampleEnum> package = 8; + map<string, SampleEnum> enum = 9; // Most recent Java reserved word + map<string, SampleEnum> null = 10; + // null is not a 'reserved word' per se but as a literal needs similar care +} diff --git a/java/core/src/test/proto/com/google/protobuf/map_test.proto b/java/core/src/test/proto/com/google/protobuf/map_test.proto index 9eb63fc3..bc2105e5 100644 --- a/java/core/src/test/proto/com/google/protobuf/map_test.proto +++ b/java/core/src/test/proto/com/google/protobuf/map_test.proto @@ -34,7 +34,6 @@ package map_test; option java_package = "map_test"; option java_outer_classname = "MapTestProto"; -option java_generate_equals_and_hash = true; message TestMap { message MessageValue { @@ -53,6 +52,8 @@ message TestMap { map<int32, EnumValue> int32_to_enum_field = 4; map<int32, MessageValue> int32_to_message_field = 5; map<string, int32> string_to_int32_field = 6; + map<uint32, int32> uint32_to_int32_field = 7; + map<int64, int32> int64_to_int32_field = 8; } // Used to test that a nested builder containing map fields will properly @@ -71,3 +72,39 @@ message BizarroTestMap { map<string, bytes> int32_to_message_field = 5; // different key and value types map<string, bytes> string_to_int32_field = 6; // same key type, different value } + +// Used to test that java reserved words can be used as protobuf field names +// Not all reserved words are tested (to avoid bloat) but instead an arbitrary +// subset of them chosen to cover various keyword categories like +// type, modifier, declaration, etc. +message ReservedAsMapField { + map<string, uint32> if = 1; + map<string, uint32> const = 2; + map<string, uint32> private = 3; + map<string, uint32> class = 4; + map<string, uint32> int = 5; + map<string, uint32> void = 6; + map<string, uint32> string = 7; // These are also proto keywords + map<string, uint32> package = 8; + map<string, uint32> enum = 9; // Most recent Java reserved word + map<string, uint32> null = 10; + // null is not a 'reserved word' per se but as a literal needs similar care +} + +message ReservedAsMapFieldWithEnumValue { + enum SampleEnum { + A = 0; + B = 1; + } + map<string, SampleEnum> if = 1; + map<string, SampleEnum> const = 2; + map<string, SampleEnum> private = 3; + map<string, SampleEnum> class = 4; + map<string, SampleEnum> int = 5; + map<string, SampleEnum> void = 6; + map<string, SampleEnum> string = 7; // These are also proto keywords + map<string, SampleEnum> package = 8; + map<string, SampleEnum> enum = 9; // Most recent Java reserved word + map<string, SampleEnum> null = 10; + // null is not a 'reserved word' per se but as a literal needs similar care +} diff --git a/java/core/src/test/proto/com/google/protobuf/test_bad_identifiers.proto b/java/core/src/test/proto/com/google/protobuf/test_bad_identifiers.proto index 2b1f65e4..d2c77936 100644 --- a/java/core/src/test/proto/com/google/protobuf/test_bad_identifiers.proto +++ b/java/core/src/test/proto/com/google/protobuf/test_bad_identifiers.proto @@ -43,7 +43,6 @@ package io_protocol_tests; option java_package = "com.google.protobuf"; option java_outer_classname = "TestBadIdentifiersProto"; -option java_generate_equals_and_hash = true; message TestMessage { optional string cached_size = 1; |