diff options
Diffstat (limited to 'java')
67 files changed, 5575 insertions, 1981 deletions
diff --git a/java/core/src/main/java/com/google/protobuf/ByteBufferWriter.java b/java/core/src/main/java/com/google/protobuf/ByteBufferWriter.java new file mode 100644 index 00000000..0cc38175 --- /dev/null +++ b/java/core/src/main/java/com/google/protobuf/ByteBufferWriter.java @@ -0,0 +1,145 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import static java.lang.Math.max; +import static java.lang.Math.min; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.lang.ref.SoftReference; +import java.nio.ByteBuffer; + +/** + * Utility class to provide efficient writing of {@link ByteBuffer}s to {@link OutputStream}s. + */ +final class ByteBufferWriter { + private ByteBufferWriter() {} + + /** + * Minimum size for a cached buffer. This prevents us from allocating buffers that are too + * small to be easily reused. + */ + // TODO(nathanmittler): tune this property or allow configuration? + private static final int MIN_CACHED_BUFFER_SIZE = 1024; + + /** + * Maximum size for a cached buffer. If a larger buffer is required, it will be allocated + * but not cached. + */ + // TODO(nathanmittler): tune this property or allow configuration? + private static final int MAX_CACHED_BUFFER_SIZE = 16 * 1024; + + /** + * The fraction of the requested buffer size under which the buffer will be reallocated. + */ + // TODO(nathanmittler): tune this property or allow configuration? + private static final float BUFFER_REALLOCATION_THRESHOLD = 0.5f; + + /** + * Keeping a soft reference to a thread-local buffer. This buffer is used for writing a + * {@link ByteBuffer} to an {@link OutputStream} when no zero-copy alternative was available. + * Using a "soft" reference since VMs may keep this reference around longer than "weak" + * (e.g. HotSpot will maintain soft references until memory pressure warrants collection). + */ + private static final ThreadLocal<SoftReference<byte[]>> BUFFER = + new ThreadLocal<SoftReference<byte[]>>(); + + /** + * For testing purposes only. Clears the cached buffer to force a new allocation on the next + * invocation. + */ + static void clearCachedBuffer() { + BUFFER.set(null); + } + + /** + * Writes the remaining content of the buffer to the given stream. The buffer {@code position} + * will remain unchanged by this method. + */ + static void write(ByteBuffer buffer, OutputStream output) throws IOException { + final int initialPos = buffer.position(); + try { + if (buffer.hasArray()) { + // Optimized write for array-backed buffers. + // Note that we're taking the risk that a malicious OutputStream could modify the array. + output.write(buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining()); + } else if (output instanceof FileOutputStream) { + // Use a channel to write out the ByteBuffer. This will automatically empty the buffer. + ((FileOutputStream) output).getChannel().write(buffer); + } else { + // Read all of the data from the buffer to an array. + // TODO(nathanmittler): Consider performance improvements for other "known" stream types. + final byte[] array = getOrCreateBuffer(buffer.remaining()); + while (buffer.hasRemaining()) { + int length = min(buffer.remaining(), array.length); + buffer.get(array, 0, length); + output.write(array, 0, length); + } + } + } finally { + // Restore the initial position. + buffer.position(initialPos); + } + } + + private static byte[] getOrCreateBuffer(int requestedSize) { + requestedSize = max(requestedSize, MIN_CACHED_BUFFER_SIZE); + + byte[] buffer = getBuffer(); + // Only allocate if we need to. + if (buffer == null || needToReallocate(requestedSize, buffer.length)) { + buffer = new byte[requestedSize]; + + // Only cache the buffer if it's not too big. + if (requestedSize <= MAX_CACHED_BUFFER_SIZE) { + setBuffer(buffer); + } + } + return buffer; + } + + private static boolean needToReallocate(int requestedSize, int bufferLength) { + // First check against just the requested length to avoid the multiply. + return bufferLength < requestedSize + && bufferLength < requestedSize * BUFFER_REALLOCATION_THRESHOLD; + } + + private static byte[] getBuffer() { + SoftReference<byte[]> sr = BUFFER.get(); + return sr == null ? null : sr.get(); + } + + private static void setBuffer(byte[] value) { + BUFFER.set(new SoftReference<byte[]>(value)); + } +} diff --git a/java/core/src/main/java/com/google/protobuf/ByteOutput.java b/java/core/src/main/java/com/google/protobuf/ByteOutput.java new file mode 100644 index 00000000..8b7b04c8 --- /dev/null +++ b/java/core/src/main/java/com/google/protobuf/ByteOutput.java @@ -0,0 +1,116 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * An output target for raw bytes. This interface provides semantics that support two types of + * writing: + * + * <p/><b>Traditional write operations:</b> + * (as defined by {@link java.io.OutputStream}) where the target method is responsible for either + * copying the data or completing the write before returning from the method call. + * + * <p/><b>Lazy write operations:</b> where the caller guarantees that it will never modify the + * provided buffer and it can therefore be considered immutable. The target method is free to + * maintain a reference to the buffer beyond the scope of the method call (e.g. until the write + * operation completes). + */ +@ExperimentalApi +public abstract class ByteOutput { + /** + * Writes a single byte. + * + * @param value the byte to be written + * @throws IOException thrown if an error occurred while writing + */ + public abstract void write(byte value) throws IOException; + + /** + * Writes a sequence of bytes. The {@link ByteOutput} must copy {@code value} if it will + * not be processed prior to the return of this method call, since {@code value} may be + * reused/altered by the caller. + * + * <p>NOTE: This method <strong>MUST NOT</strong> modify the {@code value}. Doing so is a + * programming error and will lead to data corruption which will be difficult to debug. + * + * @param value the bytes to be written + * @param offset the offset of the start of the writable range + * @param length the number of bytes to write starting from {@code offset} + * @throws IOException thrown if an error occurred while writing + */ + public abstract void write(byte[] value, int offset, int length) throws IOException; + + /** + * Writes a sequence of bytes. The {@link ByteOutput} is free to retain a reference to the value + * beyond the scope of this method call (e.g. write later) since it is considered immutable and is + * guaranteed not to change by the caller. + * + * <p>NOTE: This method <strong>MUST NOT</strong> modify the {@code value}. Doing so is a + * programming error and will lead to data corruption which will be difficult to debug. + * + * @param value the bytes to be written + * @param offset the offset of the start of the writable range + * @param length the number of bytes to write starting from {@code offset} + * @throws IOException thrown if an error occurred while writing + */ + public abstract void writeLazy(byte[] value, int offset, int length) throws IOException; + + /** + * Writes a sequence of bytes. The {@link ByteOutput} must copy {@code value} if it will + * not be processed prior to the return of this method call, since {@code value} may be + * reused/altered by the caller. + * + * <p>NOTE: This method <strong>MUST NOT</strong> modify the {@code value}. Doing so is a + * programming error and will lead to data corruption which will be difficult to debug. + * + * @param value the bytes to be written. Upon returning from this call, the {@code position} of + * this buffer will be set to the {@code limit} + * @throws IOException thrown if an error occurred while writing + */ + public abstract void write(ByteBuffer value) throws IOException; + + /** + * Writes a sequence of bytes. The {@link ByteOutput} is free to retain a reference to the value + * beyond the scope of this method call (e.g. write later) since it is considered immutable and is + * guaranteed not to change by the caller. + * + * <p>NOTE: This method <strong>MUST NOT</strong> modify the {@code value}. Doing so is a + * programming error and will lead to data corruption which will be difficult to debug. + * + * @param value the bytes to be written. Upon returning from this call, the {@code position} of + * this buffer will be set to the {@code limit} + * @throws IOException thrown if an error occurred while writing + */ + public abstract void writeLazy(ByteBuffer value) throws IOException; +} diff --git a/java/core/src/main/java/com/google/protobuf/ByteString.java b/java/core/src/main/java/com/google/protobuf/ByteString.java index 305236f3..62c94508 100644 --- a/java/core/src/main/java/com/google/protobuf/ByteString.java +++ b/java/core/src/main/java/com/google/protobuf/ByteString.java @@ -1,4 +1,32 @@ -// Copyright 2007 Google Inc. All rights reserved. +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.protobuf; @@ -15,6 +43,7 @@ import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.nio.charset.UnsupportedCharsetException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Iterator; @@ -58,6 +87,54 @@ public abstract class ByteString implements Iterable<Byte>, Serializable { * Empty {@code ByteString}. */ public static final ByteString EMPTY = new LiteralByteString(Internal.EMPTY_BYTE_ARRAY); + + /** + * An interface to efficiently copy {@code byte[]}. + * + * <p>One of the noticable costs of copying a byte[] into a new array using + * {@code System.arraycopy} is nullification of a new buffer before the copy. It has been shown + * the Hotspot VM is capable to intrisicfy {@code Arrays.copyOfRange} operation to avoid this + * expensive nullification and provide substantial performance gain. Unfortunately this does not + * hold on Android runtimes and could make the copy slightly slower due to additional code in + * the {@code Arrays.copyOfRange}. Thus we provide two different implementation for array copier + * for Hotspot and Android runtimes. + */ + private interface ByteArrayCopier { + /** + * Copies the specified range of the specified array into a new array + */ + byte[] copyFrom(byte[] bytes, int offset, int size); + } + + /** Implementation of {@code ByteArrayCopier} which uses {@link System#arraycopy}. */ + private static final class SystemByteArrayCopier implements ByteArrayCopier { + @Override + public byte[] copyFrom(byte[] bytes, int offset, int size) { + byte[] copy = new byte[size]; + System.arraycopy(bytes, offset, copy, 0, size); + return copy; + } + } + + /** Implementation of {@code ByteArrayCopier} which uses {@link Arrays#copyOfRange}. */ + private static final class ArraysByteArrayCopier implements ByteArrayCopier { + @Override + public byte[] copyFrom(byte[] bytes, int offset, int size) { + return Arrays.copyOfRange(bytes, offset, offset + size); + } + } + + private static final ByteArrayCopier byteArrayCopier; + static { + boolean isAndroid = true; + try { + Class.forName("android.content.Context"); + } catch (ClassNotFoundException e) { + isAndroid = false; + } + + byteArrayCopier = isAndroid ? new SystemByteArrayCopier() : new ArraysByteArrayCopier(); + } /** * Cached hash value. Intentionally accessed via a data race, which @@ -77,7 +154,7 @@ public abstract class ByteString implements Iterable<Byte>, Serializable { * * @param index index of byte * @return the value - * @throws ArrayIndexOutOfBoundsException {@code index < 0 or index >= size} + * @throws IndexOutOfBoundsException {@code index < 0 or index >= size} */ public abstract byte byteAt(int index); @@ -109,7 +186,7 @@ public abstract class ByteString implements Iterable<Byte>, Serializable { public byte nextByte() { try { return byteAt(position++); - } catch (ArrayIndexOutOfBoundsException e) { + } catch (IndexOutOfBoundsException e) { throw new NoSuchElementException(e.getMessage()); } } @@ -220,9 +297,7 @@ public abstract class ByteString implements Iterable<Byte>, Serializable { * @return new {@code ByteString} */ public static ByteString copyFrom(byte[] bytes, int offset, int size) { - byte[] copy = new byte[size]; - System.arraycopy(bytes, offset, copy, 0, size); - return new LiteralByteString(copy); + return new LiteralByteString(byteArrayCopier.copyFrom(bytes, offset, size)); } /** @@ -559,12 +634,7 @@ public abstract class ByteString implements Iterable<Byte>, Serializable { } /** - * Writes the complete contents of this byte string to - * the specified output stream argument. - * - * <p>It is assumed that the {@link OutputStream} will not modify the contents passed it - * it. It may be possible for a malicious {@link OutputStream} to corrupt - * the data underlying the {@link ByteString}. + * Writes a copy of the contents of this byte string to the specified output stream argument. * * @param out the output stream to which to write the data. * @throws IOException if an I/O error occurs. @@ -578,8 +648,7 @@ public abstract class ByteString implements Iterable<Byte>, Serializable { * @param sourceOffset offset within these bytes * @param numberToWrite number of bytes to write * @throws IOException if an I/O error occurs. - * @throws IndexOutOfBoundsException if an offset or size is negative or too - * large + * @throws IndexOutOfBoundsException if an offset or size is negative or too large */ final void writeTo(OutputStream out, int sourceOffset, int numberToWrite) throws IOException { @@ -597,6 +666,20 @@ public abstract class ByteString implements Iterable<Byte>, Serializable { throws IOException; /** + * Writes this {@link ByteString} to the provided {@link ByteOutput}. Calling + * this method may result in multiple operations on the target {@link ByteOutput}. + * + * <p>This method may expose internal backing buffers of the {@link ByteString} to the {@link + * ByteOutput} in order to avoid additional copying overhead. It would be possible for a malicious + * {@link ByteOutput} to corrupt the {@link ByteString}. Use with caution! + * + * @param byteOutput the output target to receive the bytes + * @throws IOException if an I/O error occurs + * @see UnsafeByteOperations#unsafeWriteTo(ByteString, ByteOutput) + */ + abstract void writeTo(ByteOutput byteOutput) throws IOException; + + /** * Constructs a read-only {@code java.nio.ByteBuffer} whose content * is equal to the contents of this byte string. * The result uses the same backing array as the byte string, if possible. @@ -1102,7 +1185,7 @@ public abstract class ByteString implements Iterable<Byte>, Serializable { * * @param index the index position to be tested * @param size the length of the array - * @throws ArrayIndexOutOfBoundsException if the index does not fall within the array. + * @throws IndexOutOfBoundsException if the index does not fall within the array. */ static void checkIndex(int index, int size) { if ((index | (size - (index + 1))) < 0) { @@ -1120,7 +1203,7 @@ public abstract class ByteString implements Iterable<Byte>, Serializable { * @param endIndex the end index of the range (exclusive) * @param size the size of the array. * @return the length of the range. - * @throws ArrayIndexOutOfBoundsException some or all of the range falls outside of the array. + * @throws IndexOutOfBoundsException some or all of the range falls outside of the array. */ static int checkRange(int startIndex, int endIndex, int size) { final int length = endIndex - startIndex; @@ -1236,6 +1319,11 @@ public abstract class ByteString implements Iterable<Byte>, Serializable { } @Override + final void writeTo(ByteOutput output) throws IOException { + output.writeLazy(bytes, getOffsetIntoBytes(), size()); + } + + @Override protected final String toStringInternal(Charset charset) { return new String(bytes, getOffsetIntoBytes(), size(), charset); } diff --git a/java/core/src/main/java/com/google/protobuf/CodedInputStream.java b/java/core/src/main/java/com/google/protobuf/CodedInputStream.java index b3118ee0..e8860651 100644 --- a/java/core/src/main/java/com/google/protobuf/CodedInputStream.java +++ b/java/core/src/main/java/com/google/protobuf/CodedInputStream.java @@ -55,7 +55,14 @@ public final class CodedInputStream { * Create a new CodedInputStream wrapping the given InputStream. */ public static CodedInputStream newInstance(final InputStream input) { - return new CodedInputStream(input); + return new CodedInputStream(input, BUFFER_SIZE); + } + + /** + * Create a new CodedInputStream wrapping the given InputStream. + */ + static CodedInputStream newInstance(final InputStream input, int bufferSize) { + return new CodedInputStream(input, bufferSize); } /** @@ -70,14 +77,14 @@ public final class CodedInputStream { */ public static CodedInputStream newInstance(final byte[] buf, final int off, final int len) { - return newInstance(buf, off, len, false); + return newInstance(buf, off, len, false /* bufferIsImmutable */); } - + /** * Create a new CodedInputStream wrapping the given byte array slice. */ - public static CodedInputStream newInstance(final byte[] buf, final int off, - final int len, boolean bufferIsImmutable) { + static CodedInputStream newInstance( + final byte[] buf, final int off, final int len, final boolean bufferIsImmutable) { CodedInputStream result = new CodedInputStream(buf, off, len, bufferIsImmutable); try { // Some uses of CodedInputStream can be more efficient if they know @@ -361,6 +368,11 @@ public final class CodedInputStream { return result; } else if (size == 0) { return ""; + } else if (size <= bufferSize) { + refillBuffer(size); + String result = new String(buffer, bufferPos, size, Internal.UTF_8); + bufferPos += size; + return result; } else { // Slow path: Build a byte array first then copy it. return new String(readRawBytesSlowPath(size), Internal.UTF_8); @@ -375,14 +387,21 @@ public final class CodedInputStream { public String readStringRequireUtf8() throws IOException { final int size = readRawVarint32(); final byte[] bytes; - int pos = bufferPos; - if (size <= (bufferSize - pos) && size > 0) { + final int oldPos = bufferPos; + final int pos; + if (size <= (bufferSize - oldPos) && size > 0) { // Fast path: We already have the bytes in a contiguous buffer, so // just copy directly from it. bytes = buffer; - bufferPos = pos + size; + bufferPos = oldPos + size; + pos = oldPos; } else if (size == 0) { return ""; + } else if (size <= bufferSize) { + refillBuffer(size); + bytes = buffer; + pos = 0; + bufferPos = pos + size; } else { // Slow path: Build a byte array first then copy it. bytes = readRawBytesSlowPath(size); @@ -869,7 +888,8 @@ public final class CodedInputStream { private static final int DEFAULT_SIZE_LIMIT = 64 << 20; // 64MB private static final int BUFFER_SIZE = 4096; - private CodedInputStream(final byte[] buffer, final int off, final int len, boolean bufferIsImmutable) { + private CodedInputStream( + final byte[] buffer, final int off, final int len, boolean bufferIsImmutable) { this.buffer = buffer; bufferSize = off + len; bufferPos = off; @@ -878,8 +898,8 @@ public final class CodedInputStream { this.bufferIsImmutable = bufferIsImmutable; } - private CodedInputStream(final InputStream input) { - buffer = new byte[BUFFER_SIZE]; + private CodedInputStream(final InputStream input, int bufferSize) { + buffer = new byte[bufferSize]; bufferSize = 0; bufferPos = 0; totalBytesRetired = 0; diff --git a/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java b/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java index d8ebad21..b92394b8 100644 --- a/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java +++ b/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java @@ -49,13 +49,17 @@ import java.util.logging.Logger; * you are writing some other format of your own design, use the latter. * * <p>This class is totally unsynchronized. - * - * @author kneton@google.com Kenton Varda */ public final class CodedOutputStream { - private static final Logger logger = Logger.getLogger(CodedOutputStream.class.getName()); + private static final int LITTLE_ENDIAN_64_SIZE = 8; + + /** + * @deprecated Use {@link #computeFixed32SizeNoTag(int)} instead. + */ + @Deprecated public static final int LITTLE_ENDIAN_32_SIZE = 4; + // TODO(dweis): Consider migrating to a ByteBuffer. private final byte[] buffer; private final int limit; @@ -77,12 +81,13 @@ public final class CodedOutputStream { * CodedOutputStream. */ static int computePreferredBufferSize(int dataLength) { - if (dataLength > DEFAULT_BUFFER_SIZE) return DEFAULT_BUFFER_SIZE; + if (dataLength > DEFAULT_BUFFER_SIZE) { + return DEFAULT_BUFFER_SIZE; + } return dataLength; } - private CodedOutputStream(final byte[] buffer, final int offset, - final int length) { + private CodedOutputStream(final byte[] buffer, final int offset, final int length) { output = null; this.buffer = buffer; position = offset; @@ -108,8 +113,7 @@ public final class CodedOutputStream { * Create a new {@code CodedOutputStream} wrapping the given * {@code OutputStream} with a given buffer size. */ - public static CodedOutputStream newInstance(final OutputStream output, - final int bufferSize) { + public static CodedOutputStream newInstance(final OutputStream output, final int bufferSize) { return new CodedOutputStream(output, new byte[bufferSize]); } @@ -131,9 +135,8 @@ public final class CodedOutputStream { * array is faster than writing to an {@code OutputStream}. See also * {@link ByteString#newCodedBuilder}. */ - public static CodedOutputStream newInstance(final byte[] flatArray, - final int offset, - final int length) { + public static CodedOutputStream newInstance( + final byte[] flatArray, final int offset, final int length) { return new CodedOutputStream(flatArray, offset, length); } @@ -147,13 +150,13 @@ public final class CodedOutputStream { /** * Create a new {@code CodedOutputStream} that writes to the given ByteBuffer. */ - public static CodedOutputStream newInstance(ByteBuffer byteBuffer, - int bufferSize) { + public static CodedOutputStream newInstance(ByteBuffer byteBuffer, int bufferSize) { return newInstance(new ByteBufferOutputStream(byteBuffer), bufferSize); } private static class ByteBufferOutputStream extends OutputStream { private final ByteBuffer byteBuffer; + public ByteBufferOutputStream(ByteBuffer byteBuffer) { this.byteBuffer = byteBuffer; } @@ -171,106 +174,120 @@ public final class CodedOutputStream { // ----------------------------------------------------------------- - /** Write a {@code double} field, including tag, to the stream. */ - public void writeDouble(final int fieldNumber, final double value) - throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED64); - writeDoubleNoTag(value); + /** Encode and write a tag. */ + public void writeTag(final int fieldNumber, final int wireType) throws IOException { + writeRawVarint32(WireFormat.makeTag(fieldNumber, wireType)); } - /** Write a {@code float} field, including tag, to the stream. */ - public void writeFloat(final int fieldNumber, final float value) - throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED32); - writeFloatNoTag(value); + /** Write an {@code int32} field, including tag, to the stream. */ + public void writeInt32(final int fieldNumber, final int value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + writeInt32NoTag(value); } - /** Write a {@code uint64} field, including tag, to the stream. */ - public void writeUInt64(final int fieldNumber, final long value) - throws IOException { + /** Write a {@code uint32} field, including tag, to the stream. */ + public void writeUInt32(final int fieldNumber, final int value) throws IOException { writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - writeUInt64NoTag(value); + writeUInt32NoTag(value); + } + + /** Write a {@code sint32} field, including tag, to the stream. */ + public void writeSInt32(final int fieldNumber, final int value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + writeSInt32NoTag(value); + } + + /** Write a {@code fixed32} field, including tag, to the stream. */ + public void writeFixed32(final int fieldNumber, final int value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED32); + writeFixed32NoTag(value); + } + + /** Write an {@code sfixed32} field, including tag, to the stream. */ + public void writeSFixed32(final int fieldNumber, final int value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED32); + writeSFixed32NoTag(value); } /** Write an {@code int64} field, including tag, to the stream. */ - public void writeInt64(final int fieldNumber, final long value) - throws IOException { + public void writeInt64(final int fieldNumber, final long value) throws IOException { writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); writeInt64NoTag(value); } - /** Write an {@code int32} field, including tag, to the stream. */ - public void writeInt32(final int fieldNumber, final int value) - throws IOException { + /** Write a {@code uint64} field, including tag, to the stream. */ + public void writeUInt64(final int fieldNumber, final long value) throws IOException { writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - writeInt32NoTag(value); + writeUInt64NoTag(value); + } + + /** Write an {@code sint64} field, including tag, to the stream. */ + public void writeSInt64(final int fieldNumber, final long value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + writeSInt64NoTag(value); } /** Write a {@code fixed64} field, including tag, to the stream. */ - public void writeFixed64(final int fieldNumber, final long value) - throws IOException { + public void writeFixed64(final int fieldNumber, final long value) throws IOException { writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED64); writeFixed64NoTag(value); } - /** Write a {@code fixed32} field, including tag, to the stream. */ - public void writeFixed32(final int fieldNumber, final int value) - throws IOException { + /** Write an {@code sfixed64} field, including tag, to the stream. */ + public void writeSFixed64(final int fieldNumber, final long value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED64); + writeSFixed64NoTag(value); + } + + /** Write a {@code float} field, including tag, to the stream. */ + public void writeFloat(final int fieldNumber, final float value) throws IOException { writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED32); - writeFixed32NoTag(value); + writeFloatNoTag(value); + } + + /** Write a {@code double} field, including tag, to the stream. */ + public void writeDouble(final int fieldNumber, final double value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED64); + writeDoubleNoTag(value); } /** Write a {@code bool} field, including tag, to the stream. */ - public void writeBool(final int fieldNumber, final boolean value) - throws IOException { + public void writeBool(final int fieldNumber, final boolean value) throws IOException { writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); writeBoolNoTag(value); } - /** Write a {@code string} field, including tag, to the stream. */ - public void writeString(final int fieldNumber, final String value) - throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeStringNoTag(value); - } - - /** Write a {@code group} field, including tag, to the stream. */ - public void writeGroup(final int fieldNumber, final MessageLite value) - throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_START_GROUP); - writeGroupNoTag(value); - writeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP); + /** + * Write an enum field, including tag, to the stream. The provided value is the numeric + * value used to represent the enum value on the wire (not the enum ordinal value). + */ + public void writeEnum(final int fieldNumber, final int value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + writeEnumNoTag(value); } - - /** Write an embedded message field, including tag, to the stream. */ - public void writeMessage(final int fieldNumber, final MessageLite value) - throws IOException { + /** Write a {@code string} field, including tag, to the stream. */ + public void writeString(final int fieldNumber, final String value) throws IOException { writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); - writeMessageNoTag(value); + writeStringNoTag(value); } - /** Write a {@code bytes} field, including tag, to the stream. */ - public void writeBytes(final int fieldNumber, final ByteString value) - throws IOException { + public void writeBytes(final int fieldNumber, final ByteString value) throws IOException { writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); writeBytesNoTag(value); } /** Write a {@code bytes} field, including tag, to the stream. */ - public void writeByteArray(final int fieldNumber, final byte[] value) - throws IOException { + public void writeByteArray(final int fieldNumber, final byte[] value) throws IOException { writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); writeByteArrayNoTag(value); } /** Write a {@code bytes} field, including tag, to the stream. */ - public void writeByteArray(final int fieldNumber, - final byte[] value, - final int offset, - final int length) - throws IOException { + public void writeByteArray( + final int fieldNumber, final byte[] value, final int offset, final int length) + throws IOException { writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); writeByteArrayNoTag(value, offset, length); } @@ -285,64 +302,100 @@ public final class CodedOutputStream { * of a ByteBuffer, you can call * {@code writeByteBuffer(fieldNumber, byteBuffer.slice())}. */ - public void writeByteBuffer(final int fieldNumber, final ByteBuffer value) - throws IOException { + public void writeByteBuffer(final int fieldNumber, final ByteBuffer value) throws IOException { writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); writeByteBufferNoTag(value); } - /** Write a {@code uint32} field, including tag, to the stream. */ - public void writeUInt32(final int fieldNumber, final int value) - throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - writeUInt32NoTag(value); + /** Write a single byte. */ + public void writeRawByte(final byte value) throws IOException { + if (position == limit) { + refreshBuffer(); + } + + buffer[position++] = value; + ++totalBytesWritten; } - /** - * Write an enum field, including tag, to the stream. Caller is responsible - * for converting the enum value to its numeric value. - */ - public void writeEnum(final int fieldNumber, final int value) - throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - writeEnumNoTag(value); + /** Write a single byte, represented by an integer value. */ + public void writeRawByte(final int value) throws IOException { + writeRawByte((byte) value); } - /** Write an {@code sfixed32} field, including tag, to the stream. */ - public void writeSFixed32(final int fieldNumber, final int value) - throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED32); - writeSFixed32NoTag(value); + /** Write an array of bytes. */ + public void writeRawBytes(final byte[] value) throws IOException { + writeRawBytes(value, 0, value.length); } - /** Write an {@code sfixed64} field, including tag, to the stream. */ - public void writeSFixed64(final int fieldNumber, final long value) - throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_FIXED64); - writeSFixed64NoTag(value); + /** Write part of an array of bytes. */ + public void writeRawBytes(final byte[] value, int offset, int length) throws IOException { + if (limit - position >= length) { + // We have room in the current buffer. + System.arraycopy(value, offset, buffer, position, length); + position += length; + totalBytesWritten += length; + } else { + // Write extends past current buffer. Fill the rest of this buffer and + // flush. + final int bytesWritten = limit - position; + System.arraycopy(value, offset, buffer, position, bytesWritten); + offset += bytesWritten; + length -= bytesWritten; + position = limit; + totalBytesWritten += bytesWritten; + refreshBuffer(); + + // Now deal with the rest. + // Since we have an output stream, this is our buffer + // and buffer offset == 0 + if (length <= limit) { + // Fits in new buffer. + System.arraycopy(value, offset, buffer, 0, length); + position = length; + } else { + // Write is very big. Let's do it all at once. + output.write(value, offset, length); + } + totalBytesWritten += length; + } } - /** Write an {@code sint32} field, including tag, to the stream. */ - public void writeSInt32(final int fieldNumber, final int value) - throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - writeSInt32NoTag(value); + /** Write a byte string. */ + public void writeRawBytes(final ByteString value) throws IOException { + writeRawBytes(value, 0, value.size()); } - /** Write an {@code sint64} field, including tag, to the stream. */ - public void writeSInt64(final int fieldNumber, final long value) - throws IOException { - writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); - writeSInt64NoTag(value); + /** + * Write a ByteBuffer. This method will write all content of the ByteBuffer + * regardless of the current position and limit (i.e., the number of bytes + * to be written is value.capacity(), not value.remaining()). Furthermore, + * this method doesn't alter the state of the passed-in ByteBuffer. Its + * position, limit, mark, etc. will remain unchanged. If you only want to + * write the remaining bytes of a ByteBuffer, you can call + * {@code writeRawBytes(byteBuffer.slice())}. + */ + public void writeRawBytes(final ByteBuffer value) throws IOException { + if (value.hasArray()) { + writeRawBytes(value.array(), value.arrayOffset(), value.capacity()); + } else { + ByteBuffer duplicated = value.duplicate(); + duplicated.clear(); + writeRawBytesInternal(duplicated); + } + } + + /** Write an embedded message field, including tag, to the stream. */ + public void writeMessage(final int fieldNumber, final MessageLite value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED); + writeMessageNoTag(value); } /** * Write a MessageSet extension field to the stream. For historical reasons, * the wire format differs from normal fields. */ - public void writeMessageSetExtension(final int fieldNumber, - final MessageLite value) - throws IOException { + public void writeMessageSetExtension(final int fieldNumber, final MessageLite value) + throws IOException { writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP); writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber); writeMessage(WireFormat.MESSAGE_SET_MESSAGE, value); @@ -353,9 +406,8 @@ public final class CodedOutputStream { * Write an unparsed MessageSet extension field to the stream. For * historical reasons, the wire format differs from normal fields. */ - public void writeRawMessageSetExtension(final int fieldNumber, - final ByteString value) - throws IOException { + public void writeRawMessageSetExtension(final int fieldNumber, final ByteString value) + throws IOException { writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP); writeUInt32(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber); writeBytes(WireFormat.MESSAGE_SET_MESSAGE, value); @@ -364,19 +416,34 @@ public final class CodedOutputStream { // ----------------------------------------------------------------- - /** Write a {@code double} field to the stream. */ - public void writeDoubleNoTag(final double value) throws IOException { - writeRawLittleEndian64(Double.doubleToRawLongBits(value)); + /** Write an {@code int32} field to the stream. */ + public void writeInt32NoTag(final int value) throws IOException { + if (value >= 0) { + writeRawVarint32(value); + } else { + // Must sign-extend. + writeRawVarint64(value); + } } - /** Write a {@code float} field to the stream. */ - public void writeFloatNoTag(final float value) throws IOException { - writeRawLittleEndian32(Float.floatToRawIntBits(value)); + /** Write a {@code uint32} field to the stream. */ + public void writeUInt32NoTag(final int value) throws IOException { + writeRawVarint32(value); } - /** Write a {@code uint64} field to the stream. */ - public void writeUInt64NoTag(final long value) throws IOException { - writeRawVarint64(value); + /** Write a {@code sint32} field to the stream. */ + public void writeSInt32NoTag(final int value) throws IOException { + writeRawVarint32(encodeZigZag32(value)); + } + + /** Write a {@code fixed32} field to the stream. */ + public void writeFixed32NoTag(final int value) throws IOException { + writeRawLittleEndian32(value); + } + + /** Write a {@code sfixed32} field to the stream. */ + public void writeSFixed32NoTag(final int value) throws IOException { + writeRawLittleEndian32(value); } /** Write an {@code int64} field to the stream. */ @@ -384,14 +451,14 @@ public final class CodedOutputStream { writeRawVarint64(value); } - /** Write an {@code int32} field to the stream. */ - public void writeInt32NoTag(final int value) throws IOException { - if (value >= 0) { - writeRawVarint32(value); - } else { - // Must sign-extend. - writeRawVarint64(value); - } + /** Write a {@code uint64} field to the stream. */ + public void writeUInt64NoTag(final long value) throws IOException { + writeRawVarint64(value); + } + + /** Write a {@code sint64} field to the stream. */ + public void writeSInt64NoTag(final long value) throws IOException { + writeRawVarint64(encodeZigZag64(value)); } /** Write a {@code fixed64} field to the stream. */ @@ -399,9 +466,19 @@ public final class CodedOutputStream { writeRawLittleEndian64(value); } - /** Write a {@code fixed32} field to the stream. */ - public void writeFixed32NoTag(final int value) throws IOException { - writeRawLittleEndian32(value); + /** Write a {@code sfixed64} field to the stream. */ + public void writeSFixed64NoTag(final long value) throws IOException { + writeRawLittleEndian64(value); + } + + /** Write a {@code float} field to the stream. */ + public void writeFloatNoTag(final float value) throws IOException { + writeRawLittleEndian32(Float.floatToRawIntBits(value)); + } + + /** Write a {@code double} field to the stream. */ + public void writeDoubleNoTag(final double value) throws IOException { + writeRawLittleEndian64(Double.doubleToRawLongBits(value)); } /** Write a {@code bool} field to the stream. */ @@ -409,6 +486,14 @@ public final class CodedOutputStream { writeRawByte(value ? 1 : 0); } + /** + * Write an enum field to the stream. The provided value is the numeric + * value used to represent the enum value on the wire (not the enum ordinal value). + */ + public void writeEnumNoTag(final int value) throws IOException { + writeInt32NoTag(value); + } + /** Write a {@code string} field to the stream. */ // TODO(dweis): Document behavior on ill-formed UTF-16 input. public void writeStringNoTag(final String value) throws IOException { @@ -421,89 +506,6 @@ public final class CodedOutputStream { } } - /** Write a {@code string} field to the stream. */ - private void inefficientWriteStringNoTag(final String value) throws IOException { - // Unfortunately there does not appear to be any way to tell Java to encode - // UTF-8 directly into our buffer, so we have to let it create its own byte - // array and then copy. - // TODO(dweis): Consider using nio Charset methods instead. - final byte[] bytes = value.getBytes(Internal.UTF_8); - writeRawVarint32(bytes.length); - writeRawBytes(bytes); - } - - /** - * Write a {@code string} field to the stream efficiently. If the {@code string} is malformed, - * this method rolls back its changes and throws an {@link UnpairedSurrogateException} with the - * intent that the caller will catch and retry with {@link #inefficientWriteStringNoTag(String)}. - * - * @param value the string to write to the stream - * - * @throws UnpairedSurrogateException when {@code value} is ill-formed UTF-16. - */ - private void efficientWriteStringNoTag(final String value) throws IOException { - // UTF-8 byte length of the string is at least its UTF-16 code unit length (value.length()), - // and at most 3 times of it. We take advantage of this in both branches below. - final int maxLength = value.length() * Utf8.MAX_BYTES_PER_CHAR; - final int maxLengthVarIntSize = computeRawVarint32Size(maxLength); - - // If we are streaming and the potential length is too big to fit in our buffer, we take the - // slower path. Otherwise, we're good to try the fast path. - if (output != null && maxLengthVarIntSize + maxLength > limit - position) { - // Allocate a byte[] that we know can fit the string and encode into it. String.getBytes() - // does the same internally and then does *another copy* to return a byte[] of exactly the - // right size. We can skip that copy and just writeRawBytes up to the actualLength of the - // UTF-8 encoded bytes. - final byte[] encodedBytes = new byte[maxLength]; - int actualLength = Utf8.encode(value, encodedBytes, 0, maxLength); - writeRawVarint32(actualLength); - writeRawBytes(encodedBytes, 0, actualLength); - } else { - // Optimize for the case where we know this length results in a constant varint length as this - // saves a pass for measuring the length of the string. - final int minLengthVarIntSize = computeRawVarint32Size(value.length()); - int oldPosition = position; - final int length; - try { - if (minLengthVarIntSize == maxLengthVarIntSize) { - position = oldPosition + minLengthVarIntSize; - int newPosition = Utf8.encode(value, buffer, position, limit - position); - // Since this class is stateful and tracks the position, we rewind and store the state, - // prepend the length, then reset it back to the end of the string. - position = oldPosition; - length = newPosition - oldPosition - minLengthVarIntSize; - writeRawVarint32(length); - position = newPosition; - } else { - length = Utf8.encodedLength(value); - writeRawVarint32(length); - position = Utf8.encode(value, buffer, position, limit - position); - } - } catch (UnpairedSurrogateException e) { - // Be extra careful and restore the original position for retrying the write with the less - // efficient path. - position = oldPosition; - throw e; - } catch (ArrayIndexOutOfBoundsException e) { - throw new OutOfSpaceException(e); - } - totalBytesWritten += length; - } - } - - /** Write a {@code group} field to the stream. */ - public void writeGroupNoTag(final MessageLite value) throws IOException { - value.writeTo(this); - } - - - /** Write an embedded message field to the stream. */ - public void writeMessageNoTag(final MessageLite value) throws IOException { - writeRawVarint32(value.getSerializedSize()); - value.writeTo(this); - } - - /** Write a {@code bytes} field to the stream. */ public void writeBytesNoTag(final ByteString value) throws IOException { writeRawVarint32(value.size()); @@ -516,86 +518,53 @@ public final class CodedOutputStream { writeRawBytes(value); } - /** Write a {@code bytes} field to the stream. */ - public void writeByteArrayNoTag(final byte[] value, - final int offset, - final int length) throws IOException { - writeRawVarint32(length); - writeRawBytes(value, offset, length); + /** Write an embedded message field to the stream. */ + public void writeMessageNoTag(final MessageLite value) throws IOException { + writeRawVarint32(value.getSerializedSize()); + value.writeTo(this); } + // ================================================================= + // ================================================================= + /** - * Write a {@code bytes} field to the stream. This method will write all - * content of the ByteBuffer regardless of the current position and limit - * (i.e., the number of bytes to be written is value.capacity(), not - * value.remaining()). Furthermore, this method doesn't alter the state of - * the passed-in ByteBuffer. Its position, limit, mark, etc. will remain - * unchanged. If you only want to write the remaining bytes of a ByteBuffer, - * you can call {@code writeByteBufferNoTag(byteBuffer.slice())}. + * Compute the number of bytes that would be needed to encode an + * {@code int32} field, including tag. */ - public void writeByteBufferNoTag(final ByteBuffer value) throws IOException { - writeRawVarint32(value.capacity()); - writeRawBytes(value); - } - - /** Write a {@code uint32} field to the stream. */ - public void writeUInt32NoTag(final int value) throws IOException { - writeRawVarint32(value); + public static int computeInt32Size(final int fieldNumber, final int value) { + return computeTagSize(fieldNumber) + computeInt32SizeNoTag(value); } /** - * Write an enum field to the stream. Caller is responsible - * for converting the enum value to its numeric value. + * Compute the number of bytes that would be needed to encode a + * {@code uint32} field, including tag. */ - public void writeEnumNoTag(final int value) throws IOException { - writeInt32NoTag(value); - } - - /** Write an {@code sfixed32} field to the stream. */ - public void writeSFixed32NoTag(final int value) throws IOException { - writeRawLittleEndian32(value); - } - - /** Write an {@code sfixed64} field to the stream. */ - public void writeSFixed64NoTag(final long value) throws IOException { - writeRawLittleEndian64(value); - } - - /** Write an {@code sint32} field to the stream. */ - public void writeSInt32NoTag(final int value) throws IOException { - writeRawVarint32(encodeZigZag32(value)); - } - - /** Write an {@code sint64} field to the stream. */ - public void writeSInt64NoTag(final long value) throws IOException { - writeRawVarint64(encodeZigZag64(value)); + public static int computeUInt32Size(final int fieldNumber, final int value) { + return computeTagSize(fieldNumber) + computeUInt32SizeNoTag(value); } - // ================================================================= - /** - * Compute the number of bytes that would be needed to encode a - * {@code double} field, including tag. + * Compute the number of bytes that would be needed to encode an + * {@code sint32} field, including tag. */ - public static int computeDoubleSize(final int fieldNumber, - final double value) { - return computeTagSize(fieldNumber) + computeDoubleSizeNoTag(value); + public static int computeSInt32Size(final int fieldNumber, final int value) { + return computeTagSize(fieldNumber) + computeSInt32SizeNoTag(value); } /** * Compute the number of bytes that would be needed to encode a - * {@code float} field, including tag. + * {@code fixed32} field, including tag. */ - public static int computeFloatSize(final int fieldNumber, final float value) { - return computeTagSize(fieldNumber) + computeFloatSizeNoTag(value); + public static int computeFixed32Size(final int fieldNumber, final int value) { + return computeTagSize(fieldNumber) + computeFixed32SizeNoTag(value); } /** - * Compute the number of bytes that would be needed to encode a - * {@code uint64} field, including tag. + * Compute the number of bytes that would be needed to encode an + * {@code sfixed32} field, including tag. */ - public static int computeUInt64Size(final int fieldNumber, final long value) { - return computeTagSize(fieldNumber) + computeUInt64SizeNoTag(value); + public static int computeSFixed32Size(final int fieldNumber, final int value) { + return computeTagSize(fieldNumber) + computeSFixed32SizeNoTag(value); } /** @@ -607,73 +576,83 @@ public final class CodedOutputStream { } /** + * Compute the number of bytes that would be needed to encode a + * {@code uint64} field, including tag. + */ + public static int computeUInt64Size(final int fieldNumber, final long value) { + return computeTagSize(fieldNumber) + computeUInt64SizeNoTag(value); + } + + /** * Compute the number of bytes that would be needed to encode an - * {@code int32} field, including tag. + * {@code sint64} field, including tag. */ - public static int computeInt32Size(final int fieldNumber, final int value) { - return computeTagSize(fieldNumber) + computeInt32SizeNoTag(value); + public static int computeSInt64Size(final int fieldNumber, final long value) { + return computeTagSize(fieldNumber) + computeSInt64SizeNoTag(value); } /** * Compute the number of bytes that would be needed to encode a * {@code fixed64} field, including tag. */ - public static int computeFixed64Size(final int fieldNumber, - final long value) { + public static int computeFixed64Size(final int fieldNumber, final long value) { return computeTagSize(fieldNumber) + computeFixed64SizeNoTag(value); } /** - * Compute the number of bytes that would be needed to encode a - * {@code fixed32} field, including tag. + * Compute the number of bytes that would be needed to encode an + * {@code sfixed64} field, including tag. */ - public static int computeFixed32Size(final int fieldNumber, - final int value) { - return computeTagSize(fieldNumber) + computeFixed32SizeNoTag(value); + public static int computeSFixed64Size(final int fieldNumber, final long value) { + return computeTagSize(fieldNumber) + computeSFixed64SizeNoTag(value); } /** * Compute the number of bytes that would be needed to encode a - * {@code bool} field, including tag. + * {@code float} field, including tag. */ - public static int computeBoolSize(final int fieldNumber, - final boolean value) { - return computeTagSize(fieldNumber) + computeBoolSizeNoTag(value); + public static int computeFloatSize(final int fieldNumber, final float value) { + return computeTagSize(fieldNumber) + computeFloatSizeNoTag(value); } /** * Compute the number of bytes that would be needed to encode a - * {@code string} field, including tag. + * {@code double} field, including tag. */ - public static int computeStringSize(final int fieldNumber, - final String value) { - return computeTagSize(fieldNumber) + computeStringSizeNoTag(value); + public static int computeDoubleSize(final int fieldNumber, final double value) { + return computeTagSize(fieldNumber) + computeDoubleSizeNoTag(value); } /** * Compute the number of bytes that would be needed to encode a - * {@code group} field, including tag. + * {@code bool} field, including tag. */ - public static int computeGroupSize(final int fieldNumber, - final MessageLite value) { - return computeTagSize(fieldNumber) * 2 + computeGroupSizeNoTag(value); + public static int computeBoolSize(final int fieldNumber, final boolean value) { + return computeTagSize(fieldNumber) + computeBoolSizeNoTag(value); } /** * Compute the number of bytes that would be needed to encode an - * embedded message field, including tag. + * enum field, including tag. The provided value is the numeric + * value used to represent the enum value on the wire (not the enum ordinal value). */ - public static int computeMessageSize(final int fieldNumber, - final MessageLite value) { - return computeTagSize(fieldNumber) + computeMessageSizeNoTag(value); + public static int computeEnumSize(final int fieldNumber, final int value) { + return computeTagSize(fieldNumber) + computeEnumSizeNoTag(value); + } + + /** + * Compute the number of bytes that would be needed to encode a + * {@code string} field, including tag. + */ + public static int computeStringSize(final int fieldNumber, final String value) { + return computeTagSize(fieldNumber) + computeStringSizeNoTag(value); } /** * Compute the number of bytes that would be needed to encode a * {@code bytes} field, including tag. */ - public static int computeBytesSize(final int fieldNumber, - final ByteString value) { + public static int computeBytesSize(final int fieldNumber, final ByteString value) { return computeTagSize(fieldNumber) + computeBytesSizeNoTag(value); } @@ -681,8 +660,7 @@ public final class CodedOutputStream { * Compute the number of bytes that would be needed to encode a * {@code bytes} field, including tag. */ - public static int computeByteArraySize(final int fieldNumber, - final byte[] value) { + public static int computeByteArraySize(final int fieldNumber, final byte[] value) { return computeTagSize(fieldNumber) + computeByteArraySizeNoTag(value); } @@ -690,8 +668,7 @@ public final class CodedOutputStream { * Compute the number of bytes that would be needed to encode a * {@code bytes} field, including tag. */ - public static int computeByteBufferSize(final int fieldNumber, - final ByteBuffer value) { + public static int computeByteBufferSize(final int fieldNumber, final ByteBuffer value) { return computeTagSize(fieldNumber) + computeByteBufferSizeNoTag(value); } @@ -699,114 +676,111 @@ public final class CodedOutputStream { * Compute the number of bytes that would be needed to encode an * embedded message in lazy field, including tag. */ - public static int computeLazyFieldSize(final int fieldNumber, - final LazyFieldLite value) { + public static int computeLazyFieldSize(final int fieldNumber, final LazyFieldLite value) { return computeTagSize(fieldNumber) + computeLazyFieldSizeNoTag(value); } /** - * Compute the number of bytes that would be needed to encode a - * {@code uint32} field, including tag. + * Compute the number of bytes that would be needed to encode an + * embedded message field, including tag. */ - public static int computeUInt32Size(final int fieldNumber, final int value) { - return computeTagSize(fieldNumber) + computeUInt32SizeNoTag(value); + public static int computeMessageSize(final int fieldNumber, final MessageLite value) { + return computeTagSize(fieldNumber) + computeMessageSizeNoTag(value); } /** - * Compute the number of bytes that would be needed to encode an - * enum field, including tag. Caller is responsible for converting the - * enum value to its numeric value. + * Compute the number of bytes that would be needed to encode a + * MessageSet extension to the stream. For historical reasons, + * the wire format differs from normal fields. */ - public static int computeEnumSize(final int fieldNumber, final int value) { - return computeTagSize(fieldNumber) + computeEnumSizeNoTag(value); + public static int computeMessageSetExtensionSize(final int fieldNumber, final MessageLite value) { + return computeTagSize(WireFormat.MESSAGE_SET_ITEM) * 2 + + computeUInt32Size(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber) + + computeMessageSize(WireFormat.MESSAGE_SET_MESSAGE, value); } /** * Compute the number of bytes that would be needed to encode an - * {@code sfixed32} field, including tag. + * unparsed MessageSet extension field to the stream. For + * historical reasons, the wire format differs from normal fields. */ - public static int computeSFixed32Size(final int fieldNumber, - final int value) { - return computeTagSize(fieldNumber) + computeSFixed32SizeNoTag(value); + public static int computeRawMessageSetExtensionSize( + final int fieldNumber, final ByteString value) { + return computeTagSize(WireFormat.MESSAGE_SET_ITEM) * 2 + + computeUInt32Size(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber) + + computeBytesSize(WireFormat.MESSAGE_SET_MESSAGE, value); } /** * Compute the number of bytes that would be needed to encode an - * {@code sfixed64} field, including tag. + * lazily parsed MessageSet extension field to the stream. For + * historical reasons, the wire format differs from normal fields. */ - public static int computeSFixed64Size(final int fieldNumber, - final long value) { - return computeTagSize(fieldNumber) + computeSFixed64SizeNoTag(value); + public static int computeLazyFieldMessageSetExtensionSize( + final int fieldNumber, final LazyFieldLite value) { + return computeTagSize(WireFormat.MESSAGE_SET_ITEM) * 2 + + computeUInt32Size(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber) + + computeLazyFieldSize(WireFormat.MESSAGE_SET_MESSAGE, value); } - /** - * Compute the number of bytes that would be needed to encode an - * {@code sint32} field, including tag. - */ - public static int computeSInt32Size(final int fieldNumber, final int value) { - return computeTagSize(fieldNumber) + computeSInt32SizeNoTag(value); + // ----------------------------------------------------------------- + + /** Compute the number of bytes that would be needed to encode a tag. */ + public static int computeTagSize(final int fieldNumber) { + return computeRawVarint32Size(WireFormat.makeTag(fieldNumber, 0)); } /** * Compute the number of bytes that would be needed to encode an - * {@code sint64} field, including tag. + * {@code int32} field, including tag. */ - public static int computeSInt64Size(final int fieldNumber, final long value) { - return computeTagSize(fieldNumber) + computeSInt64SizeNoTag(value); + public static int computeInt32SizeNoTag(final int value) { + if (value >= 0) { + return computeRawVarint32Size(value); + } else { + // Must sign-extend. + return 10; + } } /** * Compute the number of bytes that would be needed to encode a - * MessageSet extension to the stream. For historical reasons, - * the wire format differs from normal fields. + * {@code uint32} field. */ - public static int computeMessageSetExtensionSize( - final int fieldNumber, final MessageLite value) { - return computeTagSize(WireFormat.MESSAGE_SET_ITEM) * 2 + - computeUInt32Size(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber) + - computeMessageSize(WireFormat.MESSAGE_SET_MESSAGE, value); + public static int computeUInt32SizeNoTag(final int value) { + return computeRawVarint32Size(value); } /** * Compute the number of bytes that would be needed to encode an - * unparsed MessageSet extension field to the stream. For - * historical reasons, the wire format differs from normal fields. + * {@code sint32} field. */ - public static int computeRawMessageSetExtensionSize( - final int fieldNumber, final ByteString value) { - return computeTagSize(WireFormat.MESSAGE_SET_ITEM) * 2 + - computeUInt32Size(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber) + - computeBytesSize(WireFormat.MESSAGE_SET_MESSAGE, value); + public static int computeSInt32SizeNoTag(final int value) { + return computeRawVarint32Size(encodeZigZag32(value)); } /** - * Compute the number of bytes that would be needed to encode an - * lazily parsed MessageSet extension field to the stream. For - * historical reasons, the wire format differs from normal fields. + * Compute the number of bytes that would be needed to encode a + * {@code fixed32} field. */ - public static int computeLazyFieldMessageSetExtensionSize( - final int fieldNumber, final LazyFieldLite value) { - return computeTagSize(WireFormat.MESSAGE_SET_ITEM) * 2 + - computeUInt32Size(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber) + - computeLazyFieldSize(WireFormat.MESSAGE_SET_MESSAGE, value); + public static int computeFixed32SizeNoTag(@SuppressWarnings("unused") final int unused) { + return LITTLE_ENDIAN_32_SIZE; } - // ----------------------------------------------------------------- - /** - * Compute the number of bytes that would be needed to encode a - * {@code double} field, including tag. + * Compute the number of bytes that would be needed to encode an + * {@code sfixed32} field. */ - public static int computeDoubleSizeNoTag(final double value) { - return LITTLE_ENDIAN_64_SIZE; + public static int computeSFixed32SizeNoTag(@SuppressWarnings("unused") final int unused) { + return LITTLE_ENDIAN_32_SIZE; } /** - * Compute the number of bytes that would be needed to encode a - * {@code float} field, including tag. + * Compute the number of bytes that would be needed to encode an + * {@code int64} field, including tag. */ - public static int computeFloatSizeNoTag(final float value) { - return LITTLE_ENDIAN_32_SIZE; + public static int computeInt64SizeNoTag(final long value) { + return computeRawVarint64Size(value); } /** @@ -819,50 +793,62 @@ public final class CodedOutputStream { /** * Compute the number of bytes that would be needed to encode an - * {@code int64} field, including tag. + * {@code sint64} field. */ - public static int computeInt64SizeNoTag(final long value) { - return computeRawVarint64Size(value); + public static int computeSInt64SizeNoTag(final long value) { + return computeRawVarint64Size(encodeZigZag64(value)); } /** - * Compute the number of bytes that would be needed to encode an - * {@code int32} field, including tag. + * Compute the number of bytes that would be needed to encode a + * {@code fixed64} field. */ - public static int computeInt32SizeNoTag(final int value) { - if (value >= 0) { - return computeRawVarint32Size(value); - } else { - // Must sign-extend. - return 10; - } + public static int computeFixed64SizeNoTag(@SuppressWarnings("unused") final long unused) { + return LITTLE_ENDIAN_64_SIZE; } /** - * Compute the number of bytes that would be needed to encode a - * {@code fixed64} field. + * Compute the number of bytes that would be needed to encode an + * {@code sfixed64} field. */ - public static int computeFixed64SizeNoTag(final long value) { + public static int computeSFixed64SizeNoTag(@SuppressWarnings("unused") final long unused) { return LITTLE_ENDIAN_64_SIZE; } /** * Compute the number of bytes that would be needed to encode a - * {@code fixed32} field. + * {@code float} field, including tag. */ - public static int computeFixed32SizeNoTag(final int value) { + public static int computeFloatSizeNoTag(@SuppressWarnings("unused") final float unused) { return LITTLE_ENDIAN_32_SIZE; } /** * Compute the number of bytes that would be needed to encode a + * {@code double} field, including tag. + */ + public static int computeDoubleSizeNoTag(@SuppressWarnings("unused") final double unused) { + return LITTLE_ENDIAN_64_SIZE; + } + + /** + * Compute the number of bytes that would be needed to encode a * {@code bool} field. */ - public static int computeBoolSizeNoTag(final boolean value) { + public static int computeBoolSizeNoTag(@SuppressWarnings("unused") final boolean unused) { return 1; } /** + * Compute the number of bytes that would be needed to encode an enum field. + * The provided value is the numeric value used to represent the enum value on the wire + * (not the enum ordinal value). + */ + public static int computeEnumSizeNoTag(final int value) { + return computeInt32SizeNoTag(value); + } + + /** * Compute the number of bytes that would be needed to encode a * {@code string} field. */ @@ -880,23 +866,6 @@ public final class CodedOutputStream { } /** - * Compute the number of bytes that would be needed to encode a - * {@code group} field. - */ - public static int computeGroupSizeNoTag(final MessageLite value) { - return value.getSerializedSize(); - } - - /** - * Compute the number of bytes that would be needed to encode an embedded - * message field. - */ - public static int computeMessageSizeNoTag(final MessageLite value) { - final int size = value.getSerializedSize(); - return computeRawVarint32Size(size) + size; - } - - /** * Compute the number of bytes that would be needed to encode an embedded * message stored in lazy field. */ @@ -910,8 +879,7 @@ public final class CodedOutputStream { * {@code bytes} field. */ public static int computeBytesSizeNoTag(final ByteString value) { - return computeRawVarint32Size(value.size()) + - value.size(); + return computeRawVarint32Size(value.size()) + value.size(); } /** @@ -931,72 +899,47 @@ public final class CodedOutputStream { } /** - * Compute the number of bytes that would be needed to encode a - * {@code uint32} field. - */ - public static int computeUInt32SizeNoTag(final int value) { - return computeRawVarint32Size(value); - } - - /** - * Compute the number of bytes that would be needed to encode an enum field. - * Caller is responsible for converting the enum value to its numeric value. - */ - public static int computeEnumSizeNoTag(final int value) { - return computeInt32SizeNoTag(value); - } - - /** - * Compute the number of bytes that would be needed to encode an - * {@code sfixed32} field. - */ - public static int computeSFixed32SizeNoTag(final int value) { - return LITTLE_ENDIAN_32_SIZE; - } - - /** - * Compute the number of bytes that would be needed to encode an - * {@code sfixed64} field. + * Compute the number of bytes that would be needed to encode an embedded + * message field. */ - public static int computeSFixed64SizeNoTag(final long value) { - return LITTLE_ENDIAN_64_SIZE; + public static int computeMessageSizeNoTag(final MessageLite value) { + final int size = value.getSerializedSize(); + return computeRawVarint32Size(size) + size; } /** - * Compute the number of bytes that would be needed to encode an - * {@code sint32} field. + * Encode a ZigZag-encoded 32-bit value. ZigZag encodes signed integers + * into values that can be efficiently encoded with varint. (Otherwise, + * negative values must be sign-extended to 64 bits to be varint encoded, + * thus always taking 10 bytes on the wire.) + * + * @param n A signed 32-bit integer. + * @return An unsigned 32-bit integer, stored in a signed int because + * Java has no explicit unsigned support. */ - public static int computeSInt32SizeNoTag(final int value) { - return computeRawVarint32Size(encodeZigZag32(value)); + public static int encodeZigZag32(final int n) { + // Note: the right-shift must be arithmetic + return (n << 1) ^ (n >> 31); } /** - * Compute the number of bytes that would be needed to encode an - * {@code sint64} field. + * Encode a ZigZag-encoded 64-bit value. ZigZag encodes signed integers + * into values that can be efficiently encoded with varint. (Otherwise, + * negative values must be sign-extended to 64 bits to be varint encoded, + * thus always taking 10 bytes on the wire.) + * + * @param n A signed 64-bit integer. + * @return An unsigned 64-bit integer, stored in a signed int because + * Java has no explicit unsigned support. */ - public static int computeSInt64SizeNoTag(final long value) { - return computeRawVarint64Size(encodeZigZag64(value)); + public static long encodeZigZag64(final long n) { + // Note: the right-shift must be arithmetic + return (n << 1) ^ (n >> 63); } // ================================================================= /** - * Internal helper that writes the current buffer to the output. The - * buffer position is reset to its initial value when this returns. - */ - private void refreshBuffer() throws IOException { - if (output == null) { - // We're writing to a single buffer. - throw new OutOfSpaceException(); - } - - // Since we have an output stream, this is our buffer - // and buffer offset == 0 - output.write(buffer, 0, position); - position = 0; - } - - /** * Flushes the stream and forces any buffered bytes to be written. This * does not flush the underlying OutputStream. */ @@ -1015,8 +958,8 @@ public final class CodedOutputStream { return limit - position; } else { throw new UnsupportedOperationException( - "spaceLeft() can only be called on CodedOutputStreams that are " + - "writing to a flat array."); + "spaceLeft() can only be called on CodedOutputStreams that are " + + "writing to a flat array."); } } @@ -1029,8 +972,7 @@ public final class CodedOutputStream { */ public void checkNoSpaceLeft() { if (spaceLeft() != 0) { - throw new IllegalStateException( - "Did not write as much data as expected."); + throw new IllegalStateException("Did not write as much data as expected."); } } @@ -1063,53 +1005,96 @@ public final class CodedOutputStream { return totalBytesWritten; } - /** Write a single byte. */ - public void writeRawByte(final byte value) throws IOException { - if (position == limit) { - refreshBuffer(); - } - - buffer[position++] = value; - ++totalBytesWritten; - } + // ================================================================= - /** Write a single byte, represented by an integer value. */ - public void writeRawByte(final int value) throws IOException { - writeRawByte((byte) value); - } + /** + * Internal helper that writes the current buffer to the output. The + * buffer position is reset to its initial value when this returns. + */ + private void refreshBuffer() throws IOException { + if (output == null) { + // We're writing to a single buffer. + throw new OutOfSpaceException(); + } - /** Write a byte string. */ - public void writeRawBytes(final ByteString value) throws IOException { - writeRawBytes(value, 0, value.size()); + // Since we have an output stream, this is our buffer + // and buffer offset == 0 + output.write(buffer, 0, position); + position = 0; } - /** Write an array of bytes. */ - public void writeRawBytes(final byte[] value) throws IOException { - writeRawBytes(value, 0, value.length); + /** Write a {@code string} field to the stream. */ + private void inefficientWriteStringNoTag(final String value) throws IOException { + // Unfortunately there does not appear to be any way to tell Java to encode + // UTF-8 directly into our buffer, so we have to let it create its own byte + // array and then copy. + // TODO(dweis): Consider using nio Charset methods instead. + final byte[] bytes = value.getBytes(Internal.UTF_8); + writeRawVarint32(bytes.length); + writeRawBytes(bytes); } /** - * Write a ByteBuffer. This method will write all content of the ByteBuffer - * regardless of the current position and limit (i.e., the number of bytes - * to be written is value.capacity(), not value.remaining()). Furthermore, - * this method doesn't alter the state of the passed-in ByteBuffer. Its - * position, limit, mark, etc. will remain unchanged. If you only want to - * write the remaining bytes of a ByteBuffer, you can call - * {@code writeRawBytes(byteBuffer.slice())}. + * Write a {@code string} field to the stream efficiently. If the {@code string} is malformed, + * this method rolls back its changes and throws an {@link UnpairedSurrogateException} with the + * intent that the caller will catch and retry with {@link #inefficientWriteStringNoTag(String)}. + * + * @param value the string to write to the stream + * + * @throws UnpairedSurrogateException when {@code value} is ill-formed UTF-16. */ - public void writeRawBytes(final ByteBuffer value) throws IOException { - if (value.hasArray()) { - writeRawBytes(value.array(), value.arrayOffset(), value.capacity()); + private void efficientWriteStringNoTag(final String value) throws IOException { + // UTF-8 byte length of the string is at least its UTF-16 code unit length (value.length()), + // and at most 3 times of it. We take advantage of this in both branches below. + final int maxLength = value.length() * Utf8.MAX_BYTES_PER_CHAR; + final int maxLengthVarIntSize = computeRawVarint32Size(maxLength); + + // If we are streaming and the potential length is too big to fit in our buffer, we take the + // slower path. Otherwise, we're good to try the fast path. + if (output != null && maxLengthVarIntSize + maxLength > limit - position) { + // Allocate a byte[] that we know can fit the string and encode into it. String.getBytes() + // does the same internally and then does *another copy* to return a byte[] of exactly the + // right size. We can skip that copy and just writeRawBytes up to the actualLength of the + // UTF-8 encoded bytes. + final byte[] encodedBytes = new byte[maxLength]; + int actualLength = Utf8.encode(value, encodedBytes, 0, maxLength); + writeRawVarint32(actualLength); + writeRawBytes(encodedBytes, 0, actualLength); } else { - ByteBuffer duplicated = value.duplicate(); - duplicated.clear(); - writeRawBytesInternal(duplicated); + // Optimize for the case where we know this length results in a constant varint length as this + // saves a pass for measuring the length of the string. + final int minLengthVarIntSize = computeRawVarint32Size(value.length()); + int oldPosition = position; + final int length; + try { + if (minLengthVarIntSize == maxLengthVarIntSize) { + position = oldPosition + minLengthVarIntSize; + int newPosition = Utf8.encode(value, buffer, position, limit - position); + // Since this class is stateful and tracks the position, we rewind and store the state, + // prepend the length, then reset it back to the end of the string. + position = oldPosition; + length = newPosition - oldPosition - minLengthVarIntSize; + writeRawVarint32(length); + position = newPosition; + } else { + length = Utf8.encodedLength(value); + writeRawVarint32(length); + position = Utf8.encode(value, buffer, position, limit - position); + } + } catch (UnpairedSurrogateException e) { + // Be extra careful and restore the original position for retrying the write with the less + // efficient path. + position = oldPosition; + throw e; + } catch (ArrayIndexOutOfBoundsException e) { + throw new OutOfSpaceException(e); + } + totalBytesWritten += length; } } /** Write a ByteBuffer that isn't backed by an array. */ - private void writeRawBytesInternal(final ByteBuffer value) - throws IOException { + private void writeRawBytesInternal(final ByteBuffer value) throws IOException { int length = value.remaining(); if (limit - position >= length) { // We have room in the current buffer. @@ -1143,43 +1128,29 @@ public final class CodedOutputStream { } } - /** Write part of an array of bytes. */ - public void writeRawBytes(final byte[] value, int offset, int length) - throws IOException { - if (limit - position >= length) { - // We have room in the current buffer. - System.arraycopy(value, offset, buffer, position, length); - position += length; - totalBytesWritten += length; - } else { - // Write extends past current buffer. Fill the rest of this buffer and - // flush. - final int bytesWritten = limit - position; - System.arraycopy(value, offset, buffer, position, bytesWritten); - offset += bytesWritten; - length -= bytesWritten; - position = limit; - totalBytesWritten += bytesWritten; - refreshBuffer(); + /** Write a {@code bytes} field to the stream. Visible for testing. */ + void writeByteArrayNoTag(final byte[] value, final int offset, final int length) + throws IOException { + writeRawVarint32(length); + writeRawBytes(value, offset, length); + } - // Now deal with the rest. - // Since we have an output stream, this is our buffer - // and buffer offset == 0 - if (length <= limit) { - // Fits in new buffer. - System.arraycopy(value, offset, buffer, 0, length); - position = length; - } else { - // Write is very big. Let's do it all at once. - output.write(value, offset, length); - } - totalBytesWritten += length; - } + /** + * Write a {@code bytes} field to the stream. This method will write all + * content of the ByteBuffer regardless of the current position and limit + * (i.e., the number of bytes to be written is value.capacity(), not + * value.remaining()). Furthermore, this method doesn't alter the state of + * the passed-in ByteBuffer. Its position, limit, mark, etc. will remain + * unchanged. If you only want to write the remaining bytes of a ByteBuffer, + * you can call {@code writeByteBufferNoTag(byteBuffer.slice())}. + */ + private void writeByteBufferNoTag(final ByteBuffer value) throws IOException { + writeRawVarint32(value.capacity()); + writeRawBytes(value); } /** Write part of a byte string. */ - public void writeRawBytes(final ByteString value, int offset, int length) - throws IOException { + private void writeRawBytes(final ByteString value, int offset, int length) throws IOException { if (limit - position >= length) { // We have room in the current buffer. value.copyTo(buffer, offset, position, length); @@ -1210,21 +1181,57 @@ public final class CodedOutputStream { } } - /** Encode and write a tag. */ - public void writeTag(final int fieldNumber, final int wireType) - throws IOException { - writeRawVarint32(WireFormat.makeTag(fieldNumber, wireType)); + // ================================================================= + + /** + * Write a {@code group} field, including tag, to the stream. + * + * @deprecated groups are deprecated. + */ + @Deprecated + public void writeGroup(final int fieldNumber, final MessageLite value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_START_GROUP); + writeGroupNoTag(value); + writeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP); } - /** Compute the number of bytes that would be needed to encode a tag. */ - public static int computeTagSize(final int fieldNumber) { - return computeRawVarint32Size(WireFormat.makeTag(fieldNumber, 0)); + /** + * Write a {@code group} field to the stream. + * + * @deprecated groups are deprecated. + */ + @Deprecated + public void writeGroupNoTag(final MessageLite value) throws IOException { + value.writeTo(this); + } + + /** + * Compute the number of bytes that would be needed to encode a + * {@code group} field, including tag. + * + * @deprecated groups are deprecated. + */ + @Deprecated + public static int computeGroupSize(final int fieldNumber, final MessageLite value) { + return computeTagSize(fieldNumber) * 2 + computeGroupSizeNoTag(value); + } + + /** + * Compute the number of bytes that would be needed to encode a + * {@code group} field. + */ + @Deprecated + public static int computeGroupSizeNoTag(final MessageLite value) { + return value.getSerializedSize(); } /** * Encode and write a varint. {@code value} is treated as * unsigned, so it won't be sign-extended if negative. + * + * @deprecated use {@link #writeUInt32NoTag} instead. */ + @Deprecated public void writeRawVarint32(int value) throws IOException { while (true) { if ((value & ~0x7F) == 0) { @@ -1238,95 +1245,104 @@ public final class CodedOutputStream { } /** - * Compute the number of bytes that would be needed to encode a varint. - * {@code value} is treated as unsigned, so it won't be sign-extended if - * negative. + * Encode and write a varint. + * + * @deprecated use {@link #writeUInt64NoTag} instead. */ - public static int computeRawVarint32Size(final int value) { - if ((value & (~0 << 7)) == 0) return 1; - if ((value & (~0 << 14)) == 0) return 2; - if ((value & (~0 << 21)) == 0) return 3; - if ((value & (~0 << 28)) == 0) return 4; - return 5; - } - - /** Encode and write a varint. */ + @Deprecated public void writeRawVarint64(long value) throws IOException { while (true) { if ((value & ~0x7FL) == 0) { - writeRawByte((int)value); + writeRawByte((int) value); return; } else { - writeRawByte(((int)value & 0x7F) | 0x80); + writeRawByte(((int) value & 0x7F) | 0x80); value >>>= 7; } } } - /** Compute the number of bytes that would be needed to encode a varint. */ + /** + * Compute the number of bytes that would be needed to encode a varint. + * {@code value} is treated as unsigned, so it won't be sign-extended if + * negative. + * + * @deprecated use {@link #computeUInt32SizeNoTag(int)} instead. + */ + @Deprecated + public static int computeRawVarint32Size(final int value) { + if ((value & (~0 << 7)) == 0) { + return 1; + } + if ((value & (~0 << 14)) == 0) { + return 2; + } + if ((value & (~0 << 21)) == 0) { + return 3; + } + if ((value & (~0 << 28)) == 0) { + return 4; + } + return 5; + } + + /** + * Compute the number of bytes that would be needed to encode a varint. + * + * @deprecated use {@link #computeUInt64SizeNoTag(long)} instead. + */ + @Deprecated public static int computeRawVarint64Size(long value) { // handle two popular special cases up front ... - if ((value & (~0L << 7)) == 0L) return 1; - if (value < 0L) return 10; + if ((value & (~0L << 7)) == 0L) { + return 1; + } + if (value < 0L) { + return 10; + } // ... leaving us with 8 remaining, which we can divide and conquer int n = 2; - if ((value & (~0L << 35)) != 0L) { n += 4; value >>>= 28; } - if ((value & (~0L << 21)) != 0L) { n += 2; value >>>= 14; } - if ((value & (~0L << 14)) != 0L) { n += 1; } + if ((value & (~0L << 35)) != 0L) { + n += 4; + value >>>= 28; + } + if ((value & (~0L << 21)) != 0L) { + n += 2; + value >>>= 14; + } + if ((value & (~0L << 14)) != 0L) { + n += 1; + } return n; } - /** Write a little-endian 32-bit integer. */ - public void writeRawLittleEndian32(final int value) throws IOException { - writeRawByte((value ) & 0xFF); - writeRawByte((value >> 8) & 0xFF); - writeRawByte((value >> 16) & 0xFF); - writeRawByte((value >> 24) & 0xFF); - } - - public static final int LITTLE_ENDIAN_32_SIZE = 4; - - /** Write a little-endian 64-bit integer. */ - public void writeRawLittleEndian64(final long value) throws IOException { - writeRawByte((int)(value ) & 0xFF); - writeRawByte((int)(value >> 8) & 0xFF); - writeRawByte((int)(value >> 16) & 0xFF); - writeRawByte((int)(value >> 24) & 0xFF); - writeRawByte((int)(value >> 32) & 0xFF); - writeRawByte((int)(value >> 40) & 0xFF); - writeRawByte((int)(value >> 48) & 0xFF); - writeRawByte((int)(value >> 56) & 0xFF); - } - - public static final int LITTLE_ENDIAN_64_SIZE = 8; - /** - * Encode a ZigZag-encoded 32-bit value. ZigZag encodes signed integers - * into values that can be efficiently encoded with varint. (Otherwise, - * negative values must be sign-extended to 64 bits to be varint encoded, - * thus always taking 10 bytes on the wire.) + * Write a little-endian 32-bit integer. * - * @param n A signed 32-bit integer. - * @return An unsigned 32-bit integer, stored in a signed int because - * Java has no explicit unsigned support. + * @deprecated Use {@link #writeFixed32NoTag} instead. */ - public static int encodeZigZag32(final int n) { - // Note: the right-shift must be arithmetic - return (n << 1) ^ (n >> 31); + @Deprecated + public void writeRawLittleEndian32(final int value) throws IOException { + writeRawByte((value) & 0xFF); + writeRawByte((value >> 8) & 0xFF); + writeRawByte((value >> 16) & 0xFF); + writeRawByte((value >> 24) & 0xFF); } /** - * Encode a ZigZag-encoded 64-bit value. ZigZag encodes signed integers - * into values that can be efficiently encoded with varint. (Otherwise, - * negative values must be sign-extended to 64 bits to be varint encoded, - * thus always taking 10 bytes on the wire.) + * Write a little-endian 64-bit integer. * - * @param n A signed 64-bit integer. - * @return An unsigned 64-bit integer, stored in a signed int because - * Java has no explicit unsigned support. + * @deprecated Use {@link #writeFixed64NoTag} instead. */ - public static long encodeZigZag64(final long n) { - // Note: the right-shift must be arithmetic - return (n << 1) ^ (n >> 63); + @Deprecated + public void writeRawLittleEndian64(final long value) throws IOException { + writeRawByte((int) (value) & 0xFF); + writeRawByte((int) (value >> 8) & 0xFF); + writeRawByte((int) (value >> 16) & 0xFF); + writeRawByte((int) (value >> 24) & 0xFF); + writeRawByte((int) (value >> 32) & 0xFF); + writeRawByte((int) (value >> 40) & 0xFF); + writeRawByte((int) (value >> 48) & 0xFF); + writeRawByte((int) (value >> 56) & 0xFF); } } diff --git a/java/core/src/main/java/com/google/protobuf/Descriptors.java b/java/core/src/main/java/com/google/protobuf/Descriptors.java index 5e15cfbe..e303e138 100644 --- a/java/core/src/main/java/com/google/protobuf/Descriptors.java +++ b/java/core/src/main/java/com/google/protobuf/Descriptors.java @@ -272,7 +272,7 @@ public final class Descriptors { * because a field has an undefined type or because two messages * were defined with the same name. */ - private static FileDescriptor buildFrom( + public static FileDescriptor buildFrom( final FileDescriptorProto proto, final FileDescriptor[] dependencies, final boolean allowUnknownDependencies) throws DescriptorValidationException { @@ -1123,7 +1123,7 @@ public final class Descriptors { private JavaType javaType; public FieldDescriptorProto.Type toProto() { - return FieldDescriptorProto.Type.valueOf(ordinal() + 1); + return FieldDescriptorProto.Type.forNumber(ordinal() + 1); } public JavaType getJavaType() { return javaType; } diff --git a/java/core/src/main/java/com/google/protobuf/ExperimentalApi.java b/java/core/src/main/java/com/google/protobuf/ExperimentalApi.java index 6f41fb81..3cd4c884 100644 --- a/java/core/src/main/java/com/google/protobuf/ExperimentalApi.java +++ b/java/core/src/main/java/com/google/protobuf/ExperimentalApi.java @@ -1,3 +1,33 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + package com.google.protobuf; import java.lang.annotation.Documented; diff --git a/java/core/src/main/java/com/google/protobuf/GeneratedMessage.java b/java/core/src/main/java/com/google/protobuf/GeneratedMessage.java index ceb97a4e..a50afe55 100644 --- a/java/core/src/main/java/com/google/protobuf/GeneratedMessage.java +++ b/java/core/src/main/java/com/google/protobuf/GeneratedMessage.java @@ -1019,7 +1019,9 @@ public abstract class GeneratedMessage extends AbstractMessage verifyContainingType(field); final Object value = extensions.getField(field); if (value == null) { - if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { + if (field.isRepeated()) { + return Collections.emptyList(); + } else if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { // Lacking an ExtensionRegistry, we have no way to determine the // extension's real type, so we return a DynamicMessage. return DynamicMessage.getDefaultInstance(field.getMessageType()); diff --git a/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java b/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java index 81e1862c..12a1472d 100644 --- a/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java +++ b/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java @@ -30,6 +30,7 @@ package com.google.protobuf; +import com.google.protobuf.AbstractMessageLite.Builder.LimitedInputStream; import com.google.protobuf.Internal.BooleanList; import com.google.protobuf.Internal.DoubleList; import com.google.protobuf.Internal.FloatList; @@ -39,6 +40,7 @@ import com.google.protobuf.Internal.ProtobufList; import com.google.protobuf.WireFormat.FieldType; import java.io.IOException; +import java.io.InputStream; import java.io.ObjectStreamException; import java.io.Serializable; import java.lang.reflect.InvocationTargetException; @@ -57,10 +59,7 @@ import java.util.Map; public abstract class GeneratedMessageLite< MessageType extends GeneratedMessageLite<MessageType, BuilderType>, BuilderType extends GeneratedMessageLite.Builder<MessageType, BuilderType>> - extends AbstractMessageLite - implements Serializable { - - private static final long serialVersionUID = 1L; + extends AbstractMessageLite { /** For use by generated code only. Lazily initialized to reduce allocations. */ protected UnknownFieldSetLite unknownFields = null; @@ -83,6 +82,24 @@ public abstract class GeneratedMessageLite< return (BuilderType) dynamicMethod(MethodToInvoke.NEW_BUILDER); } + /** + * A reflective toString function. This is primarily intended as a developer aid, while keeping + * binary size down. The first line of the {@code toString()} representation includes a commented + * version of {@code super.toString()} to act as an indicator that this should not be relied on + * for comparisons. + * <p> + * NOTE: This method relies on the field getter methods not being stripped or renamed by proguard. + * If they are, the fields will not be included in the returned string representation. + * <p> + * NOTE: This implementation is liable to change in the future, and should not be relied on in + * code. + */ + @Override + public String toString() { + return MessageLiteToString.toString(this, super.toString()); + } + + // The general strategy for unknown fields is to use an UnknownFieldSetLite that is treated as // mutable during the parsing constructor and immutable after. This allows us to avoid // any unnecessary intermediary allocations while reducing the generated code size. @@ -303,10 +320,9 @@ public abstract class GeneratedMessageLite< throws java.io.IOException { MessageType parsedMessage = null; try { - parsedMessage = - (MessageType) getDefaultInstanceForType().getParserForType().parsePartialFrom( - input, extensionRegistry); - } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = parsePartialFrom( + (MessageType) getDefaultInstanceForType(), input, extensionRegistry); + } catch (InvalidProtocolBufferException e) { parsedMessage = (MessageType) e.getUnfinishedMessage(); throw e; } finally { @@ -562,7 +578,6 @@ public abstract class GeneratedMessageLite< return extensions.isInitialized(); } - @Override protected final void doneParsing() { super.doneParsing(); @@ -1049,7 +1064,12 @@ public abstract class GeneratedMessageLite< * A serialized (serializable) form of the generated message. Stores the * message as a class name and a byte array. */ - static final class SerializedForm implements Serializable { + protected static final class SerializedForm implements Serializable { + + public static SerializedForm of(MessageLite message) { + return new SerializedForm(message); + } + private static final long serialVersionUID = 0L; private final String messageClassName; @@ -1093,16 +1113,6 @@ public abstract class GeneratedMessageLite< } } } - - /** - * Replaces this object in the output stream with a serialized form. - * Part of Java's serialization magic. Generated sub-classes must override - * this method by calling {@code return super.writeReplace();} - * @return a SerializedForm of this message - */ - protected Object writeReplace() throws ObjectStreamException { - return new SerializedForm(this); - } /** * Checks that the {@link Extension} is Lite and returns it as a @@ -1135,45 +1145,6 @@ public abstract class GeneratedMessageLite< message.dynamicMethod(MethodToInvoke.MAKE_IMMUTABLE); } - /** - * A static helper method for parsing a partial from input using the extension registry and the - * instance. - */ - static <T extends GeneratedMessageLite<T, ?>> T parsePartialFrom( - T instance, CodedInputStream input, ExtensionRegistryLite extensionRegistry) - throws InvalidProtocolBufferException { - try { - return (T) instance.dynamicMethod( - MethodToInvoke.PARSE_PARTIAL_FROM, input, extensionRegistry); - } catch (RuntimeException e) { - if (e.getCause() instanceof InvalidProtocolBufferException) { - throw (InvalidProtocolBufferException) e.getCause(); - } - throw e; - } - } - - /** - * A {@link Parser} implementation that delegates to the default instance. - * <p> - * For use by generated code only. - */ - protected static class DefaultInstanceBasedParser<T extends GeneratedMessageLite<T, ?>> - extends AbstractParser<T> { - - private T defaultInstance; - - public DefaultInstanceBasedParser(T defaultInstance) { - this.defaultInstance = defaultInstance; - } - - @Override - public T parsePartialFrom(CodedInputStream input, ExtensionRegistryLite extensionRegistry) - throws InvalidProtocolBufferException { - return GeneratedMessageLite.parsePartialFrom(defaultInstance, input, extensionRegistry); - } - } - protected static IntList newIntList() { return new IntArrayList(); } @@ -1269,8 +1240,218 @@ public abstract class GeneratedMessageLite< protected static <E> ProtobufList<E> emptyProtobufList() { return ProtobufArrayList.emptyList(); } - + protected static LazyStringArrayList emptyLazyStringArrayList() { return LazyStringArrayList.emptyList(); } + + /** + * A {@link Parser} implementation that delegates to the default instance. + * <p> + * For use by generated code only. + */ + protected static class DefaultInstanceBasedParser<T extends GeneratedMessageLite<T, ?>> + extends AbstractParser<T> { + + private T defaultInstance; + + public DefaultInstanceBasedParser(T defaultInstance) { + this.defaultInstance = defaultInstance; + } + + @Override + public T parsePartialFrom(CodedInputStream input, ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + return GeneratedMessageLite.parsePartialFrom(defaultInstance, input, extensionRegistry); + } + } + + /** + * A static helper method for parsing a partial from input using the extension registry and the + * instance. + */ + // TODO(dweis): Should this verify that the last tag was 0? + static <T extends GeneratedMessageLite<T, ?>> T parsePartialFrom( + T instance, CodedInputStream input, ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + T result; + try { + result = (T) instance.dynamicMethod( + MethodToInvoke.PARSE_PARTIAL_FROM, input, extensionRegistry); + } catch (RuntimeException e) { + if (e.getCause() instanceof InvalidProtocolBufferException) { + throw (InvalidProtocolBufferException) e.getCause(); + } + throw e; + } + return result; + } + + protected static <T extends GeneratedMessageLite<T, ?>> T parsePartialFrom( + T defaultInstance, + CodedInputStream input) + throws InvalidProtocolBufferException { + return parsePartialFrom(defaultInstance, input, ExtensionRegistryLite.getEmptyRegistry()); + } + + /** + * Helper method to check if message is initialized. + * + * @throws InvalidProtocolBufferException if it is not initialized. + * @return The message to check. + */ + private static <T extends GeneratedMessageLite<T, ?>> T checkMessageInitialized(T message) + throws InvalidProtocolBufferException { + if (message != null && !message.isInitialized()) { + throw message.newUninitializedMessageException() + .asInvalidProtocolBufferException() + .setUnfinishedMessage(message); + } + return message; + } + + // Validates last tag. + protected static <T extends GeneratedMessageLite<T, ?>> T parseFrom( + T defaultInstance, ByteString data) + throws InvalidProtocolBufferException { + return checkMessageInitialized( + parseFrom(defaultInstance, data, ExtensionRegistryLite.getEmptyRegistry())); + } + + // Validates last tag. + protected static <T extends GeneratedMessageLite<T, ?>> T parseFrom( + T defaultInstance, ByteString data, ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + return checkMessageInitialized(parsePartialFrom(defaultInstance, data, extensionRegistry)); + } + + // This is a special case since we want to verify that the last tag is 0. We assume we exhaust the + // ByteString. + private static <T extends GeneratedMessageLite<T, ?>> T parsePartialFrom( + T defaultInstance, ByteString data, ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + T message; + try { + CodedInputStream input = data.newCodedInput(); + message = parsePartialFrom(defaultInstance, input, extensionRegistry); + try { + input.checkLastTagWas(0); + } catch (InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(message); + } + return message; + } catch (InvalidProtocolBufferException e) { + throw e; + } + } + + // This is a special case since we want to verify that the last tag is 0. We assume we exhaust the + // ByteString. + private static <T extends GeneratedMessageLite<T, ?>> T parsePartialFrom( + T defaultInstance, byte[] data, ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + T message; + try { + CodedInputStream input = CodedInputStream.newInstance(data); + message = parsePartialFrom(defaultInstance, input, extensionRegistry); + try { + input.checkLastTagWas(0); + } catch (InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(message); + } + return message; + } catch (InvalidProtocolBufferException e) { + throw e; + } + } + + // Validates last tag. + protected static <T extends GeneratedMessageLite<T, ?>> T parseFrom( + T defaultInstance, byte[] data) + throws InvalidProtocolBufferException { + return checkMessageInitialized( + parsePartialFrom(defaultInstance, data, ExtensionRegistryLite.getEmptyRegistry())); + } + + // Validates last tag. + protected static <T extends GeneratedMessageLite<T, ?>> T parseFrom( + T defaultInstance, byte[] data, ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + return checkMessageInitialized(parsePartialFrom(defaultInstance, data, extensionRegistry)); + } + + // Does not validate last tag. + protected static <T extends GeneratedMessageLite<T, ?>> T parseFrom( + T defaultInstance, InputStream input) + throws InvalidProtocolBufferException { + return checkMessageInitialized( + parsePartialFrom(defaultInstance, CodedInputStream.newInstance(input), + ExtensionRegistryLite.getEmptyRegistry())); + } + + // Does not validate last tag. + protected static <T extends GeneratedMessageLite<T, ?>> T parseFrom( + T defaultInstance, InputStream input, ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + return checkMessageInitialized( + parsePartialFrom(defaultInstance, CodedInputStream.newInstance(input), extensionRegistry)); + } + + // Does not validate last tag. + protected static <T extends GeneratedMessageLite<T, ?>> T parseFrom( + T defaultInstance, CodedInputStream input) + throws InvalidProtocolBufferException { + return parseFrom(defaultInstance, input, ExtensionRegistryLite.getEmptyRegistry()); + } + + // Does not validate last tag. + protected static <T extends GeneratedMessageLite<T, ?>> T parseFrom( + T defaultInstance, CodedInputStream input, ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + return checkMessageInitialized( + parsePartialFrom(defaultInstance, input, extensionRegistry)); + } + + // Validates last tag. + protected static <T extends GeneratedMessageLite<T, ?>> T parseDelimitedFrom( + T defaultInstance, InputStream input) + throws InvalidProtocolBufferException { + return checkMessageInitialized( + parsePartialDelimitedFrom(defaultInstance, input, + ExtensionRegistryLite.getEmptyRegistry())); + } + + // Validates last tag. + protected static <T extends GeneratedMessageLite<T, ?>> T parseDelimitedFrom( + T defaultInstance, InputStream input, ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + return checkMessageInitialized( + parsePartialDelimitedFrom(defaultInstance, input, extensionRegistry)); + } + + private static <T extends GeneratedMessageLite<T, ?>> T parsePartialDelimitedFrom( + T defaultInstance, + InputStream input, + ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + int size; + try { + int firstByte = input.read(); + if (firstByte == -1) { + return null; + } + size = CodedInputStream.readRawVarint32(firstByte, input); + } catch (IOException e) { + throw new InvalidProtocolBufferException(e.getMessage()); + } + InputStream limitedInput = new LimitedInputStream(input, size); + CodedInputStream codedInput = CodedInputStream.newInstance(limitedInput); + T message = parsePartialFrom(defaultInstance, codedInput, extensionRegistry); + try { + codedInput.checkLastTagWas(0); + } catch (InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(message); + } + return message; + } } diff --git a/java/core/src/main/java/com/google/protobuf/Internal.java b/java/core/src/main/java/com/google/protobuf/Internal.java index e19b6dca..abf7ddd6 100644 --- a/java/core/src/main/java/com/google/protobuf/Internal.java +++ b/java/core/src/main/java/com/google/protobuf/Internal.java @@ -51,10 +51,12 @@ import java.util.Set; * * @author kenton@google.com (Kenton Varda) */ -public class Internal { +public final class Internal { - protected static final Charset UTF_8 = Charset.forName("UTF-8"); - protected static final Charset ISO_8859_1 = Charset.forName("ISO-8859-1"); + private Internal() {} + + static final Charset UTF_8 = Charset.forName("UTF-8"); + static final Charset ISO_8859_1 = Charset.forName("ISO-8859-1"); /** * Helper called by generated code to construct default values for string @@ -406,6 +408,7 @@ public class Internal { public static final CodedInputStream EMPTY_CODED_INPUT_STREAM = CodedInputStream.newInstance(EMPTY_BYTE_ARRAY); + /** * Provides an immutable view of {@code List<T>} around a {@code List<F>}. * diff --git a/java/core/src/main/java/com/google/protobuf/LazyField.java b/java/core/src/main/java/com/google/protobuf/LazyField.java index 5e0a485c..3da8b900 100644 --- a/java/core/src/main/java/com/google/protobuf/LazyField.java +++ b/java/core/src/main/java/com/google/protobuf/LazyField.java @@ -39,14 +39,14 @@ import java.util.Map.Entry; * * Most of key methods are implemented in {@link LazyFieldLite} but this class * can contain default instance of the message to provide {@code hashCode()}, - * {@code equals()} and {@code toString()}. + * {@code euqals()} and {@code toString()}. * * @author xiangl@google.com (Xiang Li) */ public class LazyField extends LazyFieldLite { /** - * Carry a message's default instance which is used by {@code hashCode()}, {@code equals()} and + * Carry a message's default instance which is used by {@code hashCode()}, {@code euqals()} and * {@code toString()}. */ private final MessageLite defaultInstance; diff --git a/java/core/src/main/java/com/google/protobuf/LazyFieldLite.java b/java/core/src/main/java/com/google/protobuf/LazyFieldLite.java index eea1fe3c..016ec20d 100644 --- a/java/core/src/main/java/com/google/protobuf/LazyFieldLite.java +++ b/java/core/src/main/java/com/google/protobuf/LazyFieldLite.java @@ -30,14 +30,26 @@ package com.google.protobuf; +import java.io.IOException; + /** * LazyFieldLite encapsulates the logic of lazily parsing message fields. It stores - * the message in a ByteString initially and then parse it on-demand. + * the message in a ByteString initially and then parses it on-demand. + * + * LazyFieldLite is thread-compatible: concurrent reads are safe once the proto that this + * LazyFieldLite is a part of is no longer being mutated by its Builder. However, explicit + * synchronization is needed under read/write situations. * - * LazyField is thread-compatible e.g. concurrent read are safe, however, - * synchronizations are needed under read/write situations. + * When a LazyFieldLite is used in the context of a MessageLite object, its behavior is considered + * to be immutable and none of the setter methods in its API are expected to be invoked. All of the + * getters are expected to be thread-safe. When used in the context of a MessageLite.Builder, + * setters can be invoked, but there is no guarantee of thread safety. + * + * TODO(yatin,dweis): Consider splitting this class's functionality and put the mutable methods + * into a separate builder class to allow us to give stronger compile-time guarantees. * - * This class is internal implementation detail, so you don't need to use it directly. + * This class is internal implementation detail of the protobuf library, so you don't need to use it + * directly. * * @author xiangl@google.com (Xiang Li) */ @@ -46,8 +58,34 @@ public class LazyFieldLite { ExtensionRegistryLite.getEmptyRegistry(); /** - * A delayed-parsed version of the bytes. When this is non-null then {@code extensionRegistry } is - * also non-null and {@code value} and {@code memoizedBytes} are null. + * The value associated with the LazyFieldLite object is stored in one or more of the following + * three fields (delayedBytes, value, memoizedBytes). They should together be interpreted as + * follows. + * 1) delayedBytes can be non-null, while value and memoizedBytes is null. The object will be in + * this state while the value for the object has not yet been parsed. + * 2) Both delayedBytes and value are non-null. The object transitions to this state as soon as + * some caller needs to access the value (by invoking getValue()). + * 3) memoizedBytes is merely an optimization for calls to LazyFieldLite.toByteString() to avoid + * recomputing the ByteString representation on each call. Instead, when the value is parsed + * from delayedBytes, we will also assign the contents of delayedBytes to memoizedBytes (since + * that is the ByteString representation of value). + * 4) Finally, if the LazyFieldLite was created directly with a parsed MessageLite value, then + * delayedBytes will be null, and memoizedBytes will be initialized only upon the first call to + * LazyFieldLite.toByteString(). + * + * Given the above conditions, any caller that needs a serialized representation of this object + * must first check if the memoizedBytes or delayedBytes ByteString is non-null and use it + * directly; if both of those are null, it can look at the parsed value field. Similarly, any + * caller that needs a parsed value must first check if the value field is already non-null, if + * not it must parse the value from delayedBytes. + */ + + /** + * A delayed-parsed version of the contents of this field. When this field is non-null, then the + * "value" field is allowed to be null until the time that the value needs to be read. + * + * When delayedBytes is non-null then {@code extensionRegistry} is required to also be non-null. + * {@code value} and {@code memoizedBytes} will be initialized lazily. */ private ByteString delayedBytes; @@ -60,12 +98,15 @@ public class LazyFieldLite { private ExtensionRegistryLite extensionRegistry; /** - * The parsed value. When this is non-null then {@code delayedBytes} will be null. + * The parsed value. When this is null and a caller needs access to the MessageLite value, then + * {@code delayedBytes} will be parsed lazily at that time. */ protected volatile MessageLite value; /** - * The memoized bytes for {@code value}. Will be null when {@code value} is null. + * The memoized bytes for {@code value}. This is an optimization for the toByteString() method to + * not have to recompute its return-value on each invocation. + * TODO(yatin): Figure out whether this optimization is actually necessary. */ private volatile ByteString memoizedBytes; @@ -230,6 +271,46 @@ public class LazyFieldLite { return; } } + + /** + * Merges another instance's contents from a stream. + * + * <p>LazyField is not thread-safe for write access. Synchronizations are needed + * under read/write situations. + */ + public void mergeFrom(CodedInputStream input, ExtensionRegistryLite extensionRegistry) + throws IOException { + if (this.containsDefaultInstance()) { + setByteString(input.readBytes(), extensionRegistry); + return; + } + + // If the other field has an extension registry but this does not, copy over the other extension + // registry. + if (this.extensionRegistry == null) { + this.extensionRegistry = extensionRegistry; + } + + // In the case that both of them are not parsed we simply concatenate the bytes to save time. In + // the (probably rare) case that they have different extension registries there is a chance that + // some of the extensions may be dropped, but the tradeoff of making this operation fast seems + // to outway the benefits of combining the extension registries, which is not normally done for + // lite protos anyways. + if (this.delayedBytes != null) { + setByteString(this.delayedBytes.concat(input.readBytes()), this.extensionRegistry); + return; + } + + // We are parsed and both contain data. We won't drop any extensions here directly, but in the + // case that the extension registries are not the same then we might in the future if we + // need to serialize and parse a message again. + try { + setValue(value.toBuilder().mergeFrom(input, extensionRegistry).build()); + } catch (InvalidProtocolBufferException e) { + // Nothing is logged and no exceptions are thrown. Clients will be unaware that a proto + // was invalid. + } + } private static MessageLite mergeValueAndBytes( MessageLite value, ByteString otherBytes, ExtensionRegistryLite extensionRegistry) { @@ -259,10 +340,10 @@ public class LazyFieldLite { * parsed. Be careful when using this method. */ public int getSerializedSize() { - if (delayedBytes != null) { - return delayedBytes.size(); - } else if (memoizedBytes != null) { + if (memoizedBytes != null) { return memoizedBytes.size(); + } else if (delayedBytes != null) { + return delayedBytes.size(); } else if (value != null) { return value.getSerializedSize(); } else { @@ -274,12 +355,12 @@ public class LazyFieldLite { * Returns a BytesString for this field in a thread-safe way. */ public ByteString toByteString() { - if (delayedBytes != null) { - return delayedBytes; - } if (memoizedBytes != null) { return memoizedBytes; } + if (delayedBytes != null) { + return delayedBytes; + } synchronized (this) { if (memoizedBytes != null) { return memoizedBytes; @@ -311,18 +392,15 @@ public class LazyFieldLite { .parseFrom(delayedBytes, extensionRegistry); this.value = parsedValue; this.memoizedBytes = delayedBytes; - this.delayedBytes = null; } else { this.value = defaultInstance; this.memoizedBytes = ByteString.EMPTY; - this.delayedBytes = null; } } catch (InvalidProtocolBufferException e) { // Nothing is logged and no exceptions are thrown. Clients will be unaware that this proto // was invalid. this.value = defaultInstance; this.memoizedBytes = ByteString.EMPTY; - this.delayedBytes = null; } } } diff --git a/java/core/src/main/java/com/google/protobuf/LazyStringArrayList.java b/java/core/src/main/java/com/google/protobuf/LazyStringArrayList.java index c3be3cca..68c430cf 100644 --- a/java/core/src/main/java/com/google/protobuf/LazyStringArrayList.java +++ b/java/core/src/main/java/com/google/protobuf/LazyStringArrayList.java @@ -30,12 +30,12 @@ package com.google.protobuf; -import java.util.Arrays; -import java.util.List; import java.util.AbstractList; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.RandomAccess; /** diff --git a/java/core/src/main/java/com/google/protobuf/MessageLiteToString.java b/java/core/src/main/java/com/google/protobuf/MessageLiteToString.java index e69de29b..2a6e0e30 100644 --- a/java/core/src/main/java/com/google/protobuf/MessageLiteToString.java +++ b/java/core/src/main/java/com/google/protobuf/MessageLiteToString.java @@ -0,0 +1,200 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** + * Helps generate {@link String} representations of {@link MessageLite} protos. + */ +final class MessageLiteToString { + /** + * Suffix for *_FIELD_NUMBER fields. This is used to reflectively detect proto fields that should + * be toString()ed. + */ + private static final String FIELD_NUMBER_NAME_SUFFIX = "_FIELD_NUMBER"; + + /** + * Returns a {@link String} representation of the {@link MessageLite} object. The first line of + * the {@code String} representation representation includes a comment string to uniquely identify + * the objcet instance. This acts as an indicator that this should not be relied on for + * comparisons. + * + * <p>For use by generated code only. + */ + static String toString(MessageLite messageLite, String commentString) { + StringBuilder buffer = new StringBuilder(); + buffer.append("# ").append(commentString); + reflectivePrintWithIndent(messageLite, buffer, 0); + return buffer.toString(); + } + + /** + * Reflectively prints the {@link MessageLite} to the buffer at given {@code indent} level. + * + * @param buffer the buffer to write to + * @param indent the number of spaces to indent the proto by + */ + private static void reflectivePrintWithIndent( + MessageLite messageLite, StringBuilder buffer, int indent) { + // Build a map of method name to method. We're looking for methods like getFoo(), hasFoo(), and + // getFooList() which might be useful for building an object's string representation. + Map<String, Method> nameToNoArgMethod = new HashMap<String, Method>(); + for (Method method : messageLite.getClass().getDeclaredMethods()) { + if (method.getParameterTypes().length == 0) { + nameToNoArgMethod.put(method.getName(), method); + } + } + + for (Field field : messageLite.getClass().getDeclaredFields()) { + String fieldName = field.getName(); + // Skip all fields that aren't in a format like "FOO_BAR_FIELD_NUMBER" + if (!fieldName.endsWith(FIELD_NUMBER_NAME_SUFFIX)) { + continue; + } + + // For "FOO_BAR_FIELD_NUMBER" his would be "FOO_BAR" + String upperUnderscore = + fieldName.substring(0, fieldName.length() - FIELD_NUMBER_NAME_SUFFIX.length()); + + // For "FOO_BAR_FIELD_NUMBER" his would be "FooBar" + String upperCamelCaseName = upperUnderscoreToUpperCamel(upperUnderscore); + + // Try to reflectively get the value and toString() the field as if it were optional. This + // only works if the method names have not be proguarded out or renamed. + Method getMethod = nameToNoArgMethod.get("get" + upperCamelCaseName); + Method hasMethod = nameToNoArgMethod.get("has" + upperCamelCaseName); + if (getMethod != null && hasMethod != null) { + if ((Boolean) GeneratedMessageLite.invokeOrDie(hasMethod, messageLite)) { + printField( + buffer, + indent, + upperUnderscore.toLowerCase(), + GeneratedMessageLite.invokeOrDie(getMethod, messageLite)); + } + continue; + } + + // Try to reflectively get the value and toString() the field as if it were repeated. This + // only works if the method names have not be proguarded out or renamed. + Method listMethod = nameToNoArgMethod.get("get" + upperCamelCaseName + "List"); + if (listMethod != null) { + printField( + buffer, + indent, + upperUnderscore.toLowerCase(), + GeneratedMessageLite.invokeOrDie(listMethod, messageLite)); + continue; + } + } + + if (messageLite instanceof GeneratedMessageLite.ExtendableMessage) { + Iterator<Map.Entry<GeneratedMessageLite.ExtensionDescriptor, Object>> iter = + ((GeneratedMessageLite.ExtendableMessage<?, ?>) messageLite).extensions.iterator(); + while (iter.hasNext()) { + Map.Entry<GeneratedMessageLite.ExtensionDescriptor, Object> entry = iter.next(); + printField(buffer, indent, "[" + entry.getKey().getNumber() + "]", entry.getValue()); + } + } + + if (((GeneratedMessageLite) messageLite).unknownFields != null) { + ((GeneratedMessageLite) messageLite).unknownFields.printWithIndent(buffer, indent); + } + } + + /** + * Formats a text proto field. + * + * <p>For use by generated code only. + * + * @param buffer the buffer to write to + * @param indent the number of spaces the proto should be indented by + * @param name the field name (in lower underscore case) + * @param object the object value of the field + */ + static final void printField(StringBuilder buffer, int indent, String name, Object object) { + if (object instanceof List<?>) { + List<?> list = (List<?>) object; + for (Object entry : list) { + printField(buffer, indent, name, entry); + } + return; + } + + buffer.append('\n'); + for (int i = 0; i < indent; i++) { + buffer.append(' '); + } + buffer.append(name); + + if (object instanceof String) { + buffer.append(": \"").append(TextFormatEscaper.escapeText((String) object)).append('"'); + } else if (object instanceof ByteString) { + buffer.append(": \"").append(TextFormatEscaper.escapeBytes((ByteString) object)).append('"'); + } else if (object instanceof GeneratedMessageLite) { + buffer.append(" {"); + reflectivePrintWithIndent((GeneratedMessageLite) object, buffer, indent + 2); + buffer.append("\n"); + for (int i = 0; i < indent; i++) { + buffer.append(' '); + } + buffer.append("}"); + } else { + buffer.append(": ").append(object.toString()); + } + } + + /** + * A Guava-less implementation of: + * {@code CaseFormat.UPPER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL, upperUnderscore)} + */ + private static String upperUnderscoreToUpperCamel(String upperUnderscore) { + String upperCamelCaseName = ""; + boolean nextCharacterShouldBeUpper = true; + for (int i = 0; i < upperUnderscore.length(); i++) { + char ch = upperUnderscore.charAt(i); + if (ch == '_') { + nextCharacterShouldBeUpper = true; + } else if (nextCharacterShouldBeUpper){ + upperCamelCaseName += Character.toUpperCase(ch); + nextCharacterShouldBeUpper = false; + } else { + upperCamelCaseName += Character.toLowerCase(ch); + } + } + return upperCamelCaseName; + } +} diff --git a/java/core/src/main/java/com/google/protobuf/NioByteString.java b/java/core/src/main/java/com/google/protobuf/NioByteString.java index f71e41b2..6163c7b1 100644 --- a/java/core/src/main/java/com/google/protobuf/NioByteString.java +++ b/java/core/src/main/java/com/google/protobuf/NioByteString.java @@ -30,15 +30,14 @@ package com.google.protobuf; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InvalidObjectException; import java.io.ObjectInputStream; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.nio.InvalidMarkException; -import java.nio.channels.Channels; import java.nio.charset.Charset; import java.util.Collections; import java.util.List; @@ -54,7 +53,7 @@ final class NioByteString extends ByteString.LeafByteString { throw new NullPointerException("buffer"); } - this.buffer = buffer.slice(); + this.buffer = buffer.slice().order(ByteOrder.nativeOrder()); } // ================================================================= @@ -119,7 +118,7 @@ final class NioByteString extends ByteString.LeafByteString { @Override public void writeTo(OutputStream out) throws IOException { - writeToInternal(out, buffer.position(), buffer.remaining()); + out.write(toByteArray()); } @Override @@ -137,14 +136,12 @@ final class NioByteString extends ByteString.LeafByteString { return; } - // Slow path - if (out instanceof FileOutputStream || numberToWrite >= 8192) { - // Use a channel to write out the ByteBuffer. - Channels.newChannel(out).write(slice(sourceOffset, sourceOffset + numberToWrite)); - } else { - // Just copy the data to an array and write it. - out.write(toByteArray()); - } + ByteBufferWriter.write(slice(sourceOffset, sourceOffset + numberToWrite), out); + } + + @Override + void writeTo(ByteOutput output) throws IOException { + output.writeLazy(buffer.slice()); } @Override @@ -159,46 +156,30 @@ final class NioByteString extends ByteString.LeafByteString { @Override protected String toStringInternal(Charset charset) { - byte[] bytes; - int offset; + final byte[] bytes; + final int offset; + final int length; if (buffer.hasArray()) { bytes = buffer.array(); offset = buffer.arrayOffset() + buffer.position(); + length = buffer.remaining(); } else { + // TODO(nathanmittler): Can we optimize this? bytes = toByteArray(); offset = 0; + length = bytes.length; } - return new String(bytes, offset, size(), charset); + return new String(bytes, offset, length, charset); } @Override public boolean isValidUtf8() { - // TODO(nathanmittler): add a ByteBuffer fork for Utf8.isValidUtf8 to avoid the copy - byte[] bytes; - int startIndex; - if (buffer.hasArray()) { - bytes = buffer.array(); - startIndex = buffer.arrayOffset() + buffer.position(); - } else { - bytes = toByteArray(); - startIndex = 0; - } - return Utf8.isValidUtf8(bytes, startIndex, startIndex + size()); + return Utf8.isValidUtf8(buffer); } @Override protected int partialIsValidUtf8(int state, int offset, int length) { - // TODO(nathanmittler): TODO add a ByteBuffer fork for Utf8.partialIsValidUtf8 to avoid the copy - byte[] bytes; - int startIndex; - if (buffer.hasArray()) { - bytes = buffer.array(); - startIndex = buffer.arrayOffset() + buffer.position(); - } else { - bytes = toByteArray(); - startIndex = 0; - } - return Utf8.partialIsValidUtf8(state, bytes, startIndex, startIndex + size()); + return Utf8.partialIsValidUtf8(state, buffer, offset, offset + length); } @Override diff --git a/java/core/src/main/java/com/google/protobuf/Parser.java b/java/core/src/main/java/com/google/protobuf/Parser.java index 3fa11c3b..6db69247 100644 --- a/java/core/src/main/java/com/google/protobuf/Parser.java +++ b/java/core/src/main/java/com/google/protobuf/Parser.java @@ -30,7 +30,6 @@ package com.google.protobuf; -import java.io.IOException; import java.io.InputStream; /** diff --git a/java/core/src/main/java/com/google/protobuf/RopeByteString.java b/java/core/src/main/java/com/google/protobuf/RopeByteString.java index 8badfabd..3f3e9bd1 100644 --- a/java/core/src/main/java/com/google/protobuf/RopeByteString.java +++ b/java/core/src/main/java/com/google/protobuf/RopeByteString.java @@ -48,10 +48,11 @@ import java.util.Stack; /** * Class to represent {@code ByteStrings} formed by concatenation of other * ByteStrings, without copying the data in the pieces. The concatenation is - * represented as a tree whose leaf nodes are each a {@link LiteralByteString}. + * represented as a tree whose leaf nodes are each a + * {@link com.google.protobuf.ByteString.LeafByteString}. * * <p>Most of the operation here is inspired by the now-famous paper <a - * href="http://www.cs.ubc.ca/local/reading/proceedings/spe91-95/spe/vol25/issue12/spe986.pdf"> + * href="https://web.archive.org/web/20060202015456/http://www.cs.ubc.ca/local/reading/proceedings/spe91-95/spe/vol25/issue12/spe986.pdf"> * BAP95 </a> Ropes: an Alternative to Strings hans-j. boehm, russ atkinson and * michael plass * @@ -139,8 +140,9 @@ final class RopeByteString extends ByteString { /** * Concatenate the given strings while performing various optimizations to * slow the growth rate of tree depth and tree node count. The result is - * either a {@link LiteralByteString} or a {@link RopeByteString} - * depending on which optimizations, if any, were applied. + * either a {@link com.google.protobuf.ByteString.LeafByteString} or a + * {@link RopeByteString} depending on which optimizations, if any, were + * applied. * * <p>Small pieces of length less than {@link * ByteString#CONCATENATE_BY_COPY_SIZE} may be copied by value here, as in @@ -294,8 +296,7 @@ final class RopeByteString extends ByteString { * * <p>Substrings of {@code length < 2} should result in at most a single * recursive call chain, terminating at a leaf node. Thus the result will be a - * {@link LiteralByteString}. {@link #RopeByteString(ByteString, - * ByteString)}. + * {@link com.google.protobuf.ByteString.LeafByteString}. * * @param beginIndex start at this index * @param endIndex the last character is the one before this index @@ -368,7 +369,7 @@ final class RopeByteString extends ByteString { @Override public List<ByteBuffer> asReadOnlyByteBufferList() { - // Walk through the list of LiteralByteString's that make up this + // Walk through the list of LeafByteString's that make up this // rope, and add each one as a read-only ByteBuffer. List<ByteBuffer> result = new ArrayList<ByteBuffer>(); PieceIterator pieces = new PieceIterator(this); @@ -400,6 +401,12 @@ final class RopeByteString extends ByteString { } @Override + void writeTo(ByteOutput output) throws IOException { + left.writeTo(output); + right.writeTo(output); + } + + @Override protected String toStringInternal(Charset charset) { return new String(toByteArray(), charset); } @@ -709,9 +716,10 @@ final class RopeByteString extends ByteString { } /** - * Returns the next item and advances one {@code LiteralByteString}. + * Returns the next item and advances one + * {@link com.google.protobuf.ByteString.LeafByteString}. * - * @return next non-empty LiteralByteString or {@code null} + * @return next non-empty LeafByteString or {@code null} */ @Override public LeafByteString next() { diff --git a/java/core/src/main/java/com/google/protobuf/SmallSortedMap.java b/java/core/src/main/java/com/google/protobuf/SmallSortedMap.java index 0674d2e2..dff19328 100644 --- a/java/core/src/main/java/com/google/protobuf/SmallSortedMap.java +++ b/java/core/src/main/java/com/google/protobuf/SmallSortedMap.java @@ -35,12 +35,12 @@ import java.util.AbstractSet; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; -import java.util.TreeMap; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; import java.util.SortedMap; +import java.util.TreeMap; /** * A custom map implementation from FieldDescriptor to Object optimized to diff --git a/java/core/src/main/java/com/google/protobuf/TextFormat.java b/java/core/src/main/java/com/google/protobuf/TextFormat.java index c99b5285..edf114fa 100644 --- a/java/core/src/main/java/com/google/protobuf/TextFormat.java +++ b/java/core/src/main/java/com/google/protobuf/TextFormat.java @@ -425,7 +425,7 @@ public final class TextFormat { case STRING: generator.print("\""); generator.print(escapeNonAscii - ? escapeText((String) value) + ? TextFormatEscaper.escapeText((String) value) : escapeDoubleQuotesAndBackslashes((String) value) .replace("\n", "\\n")); generator.print("\""); @@ -661,6 +661,14 @@ public final class TextFormat { nextToken(); } + int getLine() { + return line; + } + + int getColumn() { + return column; + } + /** Are we at the end of the input? */ public boolean atEnd() { return currentToken.length() == 0; @@ -1074,7 +1082,7 @@ public final class TextFormat { private ParseException floatParseException(final NumberFormatException e) { return parseException("Couldn't parse number: " + e.getMessage()); } - + /** * Returns a {@link UnknownFieldParseException} with the line and column * numbers of the previous token in the description, and the unknown field @@ -1133,7 +1141,7 @@ public final class TextFormat { return column; } } - + /** * Thrown when encountering an unknown field while parsing * a text format message. @@ -1257,11 +1265,14 @@ public final class TextFormat { private final boolean allowUnknownFields; private final SingularOverwritePolicy singularOverwritePolicy; + private TextFormatParseInfoTree.Builder parseInfoTreeBuilder; - private Parser(boolean allowUnknownFields, - SingularOverwritePolicy singularOverwritePolicy) { + private Parser( + boolean allowUnknownFields, SingularOverwritePolicy singularOverwritePolicy, + TextFormatParseInfoTree.Builder parseInfoTreeBuilder) { this.allowUnknownFields = allowUnknownFields; this.singularOverwritePolicy = singularOverwritePolicy; + this.parseInfoTreeBuilder = parseInfoTreeBuilder; } /** @@ -1278,6 +1289,7 @@ public final class TextFormat { private boolean allowUnknownFields = false; private SingularOverwritePolicy singularOverwritePolicy = SingularOverwritePolicy.ALLOW_SINGULAR_OVERWRITES; + private TextFormatParseInfoTree.Builder parseInfoTreeBuilder = null; /** @@ -1288,8 +1300,15 @@ public final class TextFormat { return this; } + public Builder setParseInfoTreeBuilder( + TextFormatParseInfoTree.Builder parseInfoTreeBuilder) { + this.parseInfoTreeBuilder = parseInfoTreeBuilder; + return this; + } + public Parser build() { - return new Parser(allowUnknownFields, singularOverwritePolicy); + return new Parser( + allowUnknownFields, singularOverwritePolicy, parseInfoTreeBuilder); } } @@ -1380,7 +1399,21 @@ public final class TextFormat { final ExtensionRegistry extensionRegistry, final MessageReflection.MergeTarget target) throws ParseException { + mergeField(tokenizer, extensionRegistry, target, parseInfoTreeBuilder); + } + + /** + * Parse a single field from {@code tokenizer} and merge it into + * {@code builder}. + */ + private void mergeField(final Tokenizer tokenizer, + final ExtensionRegistry extensionRegistry, + final MessageReflection.MergeTarget target, + TextFormatParseInfoTree.Builder parseTreeBuilder) + throws ParseException { FieldDescriptor field = null; + int startLine = tokenizer.getLine(); + int startColumn = tokenizer.getColumn(); final Descriptor type = target.getDescriptorForType(); ExtensionRegistry.ExtensionInfo extension = null; @@ -1472,14 +1505,51 @@ public final class TextFormat { // Handle potential ':'. if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { tokenizer.tryConsume(":"); // optional + if (parseTreeBuilder != null) { + TextFormatParseInfoTree.Builder childParseTreeBuilder = + parseTreeBuilder.getBuilderForSubMessageField(field); + consumeFieldValues(tokenizer, extensionRegistry, target, field, extension, + childParseTreeBuilder); + } else { + consumeFieldValues(tokenizer, extensionRegistry, target, field, extension, + parseTreeBuilder); + } } else { tokenizer.consume(":"); // required + consumeFieldValues( + tokenizer, extensionRegistry, target, field, extension, parseTreeBuilder); } + + if (parseTreeBuilder != null) { + parseTreeBuilder.setLocation( + field, TextFormatParseLocation.create(startLine, startColumn)); + } + + // For historical reasons, fields may optionally be separated by commas or + // semicolons. + if (!tokenizer.tryConsume(";")) { + tokenizer.tryConsume(","); + } + } + + /** + * Parse a one or more field values from {@code tokenizer} and merge it into + * {@code builder}. + */ + private void consumeFieldValues( + final Tokenizer tokenizer, + final ExtensionRegistry extensionRegistry, + final MessageReflection.MergeTarget target, + final FieldDescriptor field, + final ExtensionRegistry.ExtensionInfo extension, + final TextFormatParseInfoTree.Builder parseTreeBuilder) + throws ParseException { // Support specifying repeated field values as a comma-separated list. // Ex."foo: [1, 2, 3]" if (field.isRepeated() && tokenizer.tryConsume("[")) { while (true) { - consumeFieldValue(tokenizer, extensionRegistry, target, field, extension); + consumeFieldValue(tokenizer, extensionRegistry, target, field, extension, + parseTreeBuilder); if (tokenizer.tryConsume("]")) { // End of list. break; @@ -1487,13 +1557,8 @@ public final class TextFormat { tokenizer.consume(","); } } else { - consumeFieldValue(tokenizer, extensionRegistry, target, field, extension); - } - - // For historical reasons, fields may optionally be separated by commas or - // semicolons. - if (!tokenizer.tryConsume(";")) { - tokenizer.tryConsume(","); + consumeFieldValue( + tokenizer, extensionRegistry, target, field, extension, parseTreeBuilder); } } @@ -1506,7 +1571,8 @@ public final class TextFormat { final ExtensionRegistry extensionRegistry, final MessageReflection.MergeTarget target, final FieldDescriptor field, - final ExtensionRegistry.ExtensionInfo extension) + final ExtensionRegistry.ExtensionInfo extension, + final TextFormatParseInfoTree.Builder parseTreeBuilder) throws ParseException { Object value = null; @@ -1528,7 +1594,7 @@ public final class TextFormat { throw tokenizer.parseException( "Expected \"" + endToken + "\"."); } - mergeField(tokenizer, extensionRegistry, subField); + mergeField(tokenizer, extensionRegistry, subField, parseTreeBuilder); } value = subField.finish(); @@ -1704,52 +1770,6 @@ public final class TextFormat { // Some of these methods are package-private because Descriptors.java uses // them. - private interface ByteSequence { - int size(); - byte byteAt(int offset); - } - - /** - * Escapes bytes in the format used in protocol buffer text format, which - * is the same as the format used for C string literals. All bytes - * that are not printable 7-bit ASCII characters are escaped, as well as - * backslash, single-quote, and double-quote characters. Characters for - * which no defined short-hand escape sequence is defined will be escaped - * using 3-digit octal sequences. - */ - public static String escapeBytes(final ByteSequence input) { - final StringBuilder builder = new StringBuilder(input.size()); - for (int i = 0; i < input.size(); i++) { - final byte b = input.byteAt(i); - switch (b) { - // Java does not recognize \a or \v, apparently. - case 0x07: builder.append("\\a"); break; - case '\b': builder.append("\\b"); break; - case '\f': builder.append("\\f"); break; - case '\n': builder.append("\\n"); break; - case '\r': builder.append("\\r"); break; - case '\t': builder.append("\\t"); break; - case 0x0b: builder.append("\\v"); break; - case '\\': builder.append("\\\\"); break; - case '\'': builder.append("\\\'"); break; - case '"' : builder.append("\\\""); break; - default: - // Only ASCII characters between 0x20 (space) and 0x7e (tilde) are - // printable. Other byte values must be escaped. - if (b >= 0x20 && b <= 0x7e) { - builder.append((char) b); - } else { - builder.append('\\'); - builder.append((char) ('0' + ((b >>> 6) & 3))); - builder.append((char) ('0' + ((b >>> 3) & 7))); - builder.append((char) ('0' + (b & 7))); - } - break; - } - } - return builder.toString(); - } - /** * Escapes bytes in the format used in protocol buffer text format, which * is the same as the format used for C string literals. All bytes @@ -1758,33 +1778,15 @@ public final class TextFormat { * which no defined short-hand escape sequence is defined will be escaped * using 3-digit octal sequences. */ - public static String escapeBytes(final ByteString input) { - return escapeBytes(new ByteSequence() { - @Override - public int size() { - return input.size(); - } - @Override - public byte byteAt(int offset) { - return input.byteAt(offset); - } - }); + public static String escapeBytes(ByteString input) { + return TextFormatEscaper.escapeBytes(input); } /** * Like {@link #escapeBytes(ByteString)}, but used for byte array. */ - public static String escapeBytes(final byte[] input) { - return escapeBytes(new ByteSequence() { - @Override - public int size() { - return input.length; - } - @Override - public byte byteAt(int offset) { - return input[offset]; - } - }); + public static String escapeBytes(byte[] input) { + return TextFormatEscaper.escapeBytes(input); } /** @@ -1868,7 +1870,9 @@ public final class TextFormat { } } - return ByteString.copyFrom(result, 0, pos); + return result.length == pos + ? ByteString.wrap(result) // This reference has not been out of our control. + : ByteString.copyFrom(result, 0, pos); } /** @@ -1896,7 +1900,7 @@ public final class TextFormat { * Escape double quotes and backslashes in a String for unicode output of a message. */ public static String escapeDoubleQuotesAndBackslashes(final String input) { - return input.replace("\\", "\\\\").replace("\"", "\\\""); + return TextFormatEscaper.escapeDoubleQuotesAndBackslashes(input); } /** diff --git a/java/core/src/main/java/com/google/protobuf/TextFormatEscaper.java b/java/core/src/main/java/com/google/protobuf/TextFormatEscaper.java index e69de29b..da9ceadd 100644 --- a/java/core/src/main/java/com/google/protobuf/TextFormatEscaper.java +++ b/java/core/src/main/java/com/google/protobuf/TextFormatEscaper.java @@ -0,0 +1,137 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +/** + * Provide text format escaping support for proto2 instances. + */ +final class TextFormatEscaper { + private TextFormatEscaper() {} + + private interface ByteSequence { + int size(); + byte byteAt(int offset); + } + + /** + * Escapes bytes in the format used in protocol buffer text format, which + * is the same as the format used for C string literals. All bytes + * that are not printable 7-bit ASCII characters are escaped, as well as + * backslash, single-quote, and double-quote characters. Characters for + * which no defined short-hand escape sequence is defined will be escaped + * using 3-digit octal sequences. + */ + static String escapeBytes(final ByteSequence input) { + final StringBuilder builder = new StringBuilder(input.size()); + for (int i = 0; i < input.size(); i++) { + final byte b = input.byteAt(i); + switch (b) { + // Java does not recognize \a or \v, apparently. + case 0x07: builder.append("\\a"); break; + case '\b': builder.append("\\b"); break; + case '\f': builder.append("\\f"); break; + case '\n': builder.append("\\n"); break; + case '\r': builder.append("\\r"); break; + case '\t': builder.append("\\t"); break; + case 0x0b: builder.append("\\v"); break; + case '\\': builder.append("\\\\"); break; + case '\'': builder.append("\\\'"); break; + case '"' : builder.append("\\\""); break; + default: + // Only ASCII characters between 0x20 (space) and 0x7e (tilde) are + // printable. Other byte values must be escaped. + if (b >= 0x20 && b <= 0x7e) { + builder.append((char) b); + } else { + builder.append('\\'); + builder.append((char) ('0' + ((b >>> 6) & 3))); + builder.append((char) ('0' + ((b >>> 3) & 7))); + builder.append((char) ('0' + (b & 7))); + } + break; + } + } + return builder.toString(); + } + + /** + * Escapes bytes in the format used in protocol buffer text format, which + * is the same as the format used for C string literals. All bytes + * that are not printable 7-bit ASCII characters are escaped, as well as + * backslash, single-quote, and double-quote characters. Characters for + * which no defined short-hand escape sequence is defined will be escaped + * using 3-digit octal sequences. + */ + static String escapeBytes(final ByteString input) { + return escapeBytes(new ByteSequence() { + @Override + public int size() { + return input.size(); + } + @Override + public byte byteAt(int offset) { + return input.byteAt(offset); + } + }); + } + + /** + * Like {@link #escapeBytes(ByteString)}, but used for byte array. + */ + static String escapeBytes(final byte[] input) { + return escapeBytes(new ByteSequence() { + @Override + public int size() { + return input.length; + } + @Override + public byte byteAt(int offset) { + return input[offset]; + } + }); + } + + /** + * Like {@link #escapeBytes(ByteString)}, but escapes a text string. + * Non-ASCII characters are first encoded as UTF-8, then each byte is escaped + * individually as a 3-digit octal escape. Yes, it's weird. + */ + static String escapeText(final String input) { + return escapeBytes(ByteString.copyFromUtf8(input)); + } + + /** + * Escape double quotes and backslashes in a String for unicode output of a message. + */ + static String escapeDoubleQuotesAndBackslashes(final String input) { + return input.replace("\\", "\\\\").replace("\"", "\\\""); + } +} diff --git a/java/core/src/main/java/com/google/protobuf/TextFormatParseInfoTree.java b/java/core/src/main/java/com/google/protobuf/TextFormatParseInfoTree.java new file mode 100644 index 00000000..2ecf912e --- /dev/null +++ b/java/core/src/main/java/com/google/protobuf/TextFormatParseInfoTree.java @@ -0,0 +1,225 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import com.google.protobuf.Descriptors.FieldDescriptor; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + + +/** + * Data structure which is populated with the locations of each field value parsed from the text. + * + * <p>The locations of primary fields values are retrieved by {@code getLocation} or + * {@code getLocations}. The locations of sub message values are within nested + * {@code TextFormatParseInfoTree}s and are retrieve by {@getNestedTree} or {code @getNestedTrees}. + * + * <p>The {@code TextFormatParseInfoTree} is created by a Builder. + */ +public class TextFormatParseInfoTree { + + // Defines a mapping between each field's descriptor to the list of locations where + // its value(s) were was encountered. + private Map<FieldDescriptor, List<TextFormatParseLocation>> locationsFromField; + + // Defines a mapping between a field's descriptor to a list of TextFormatParseInfoTrees for + // sub message location information. + Map<FieldDescriptor, List<TextFormatParseInfoTree>> subtreesFromField; + + /** + * Construct a {@code TextFormatParseInfoTree}. + * + * @param locationsFromField a map of fields to location in the source code + * @param subtreeBuildersFromField a map of fields to parse tree location information builders + */ + private TextFormatParseInfoTree( + Map<FieldDescriptor, List<TextFormatParseLocation>> locationsFromField, + Map<FieldDescriptor, List<TextFormatParseInfoTree.Builder>> subtreeBuildersFromField) { + + // The maps are unmodifiable. The values in the maps are unmodifiable. + Map<FieldDescriptor, List<TextFormatParseLocation>> locs = + new HashMap<FieldDescriptor, List<TextFormatParseLocation>>(); + for (Entry<FieldDescriptor, List<TextFormatParseLocation>> kv : locationsFromField.entrySet()) { + locs.put(kv.getKey(), Collections.unmodifiableList(kv.getValue())); + } + this.locationsFromField = Collections.unmodifiableMap(locs); + + Map<FieldDescriptor, List<TextFormatParseInfoTree>> subs = + new HashMap<FieldDescriptor, List<TextFormatParseInfoTree>>(); + for (Entry<FieldDescriptor, List<Builder>> kv : subtreeBuildersFromField.entrySet()) { + List<TextFormatParseInfoTree> submessagesOfField = new ArrayList<TextFormatParseInfoTree>(); + for (Builder subBuilder : kv.getValue()) { + submessagesOfField.add(subBuilder.build()); + } + subs.put(kv.getKey(), Collections.unmodifiableList(submessagesOfField)); + } + this.subtreesFromField = Collections.unmodifiableMap(subs); + } + + /** + * Retrieve all the locations of a field. + * + * @param fieldDescriptor the the @{link FieldDescriptor} of the desired field + * @return a list of the locations of values of the field. If there are not values + * or the field doesn't exist, an empty list is returned. + */ + public List<TextFormatParseLocation> getLocations(final FieldDescriptor fieldDescriptor) { + List<TextFormatParseLocation> result = locationsFromField.get(fieldDescriptor); + return (result == null) ? Collections.<TextFormatParseLocation>emptyList() : result; + } + + /** + * Get the location in the source of a field's value. + * + * <p>Returns the {@link TextFormatParseLocation} for index-th value of the field in the parsed + * text. + * + * @param fieldDescriptor the @{link FieldDescriptor} of the desired field + * @param index the index of the value. + * @return the {@link TextFormatParseLocation} of the value + * @throws IllegalArgumentException index is out of range + */ + public TextFormatParseLocation getLocation(final FieldDescriptor fieldDescriptor, int index) { + return getFromList(getLocations(fieldDescriptor), index, fieldDescriptor); + } + + /** + * Retrieve a list of all the location information trees for a sub message field. + * + * @param fieldDescriptor the @{link FieldDescriptor} of the desired field + * @return A list of {@link TextFormatParseInfoTree} + */ + public List<TextFormatParseInfoTree> getNestedTrees(final FieldDescriptor fieldDescriptor) { + List<TextFormatParseInfoTree> result = subtreesFromField.get(fieldDescriptor); + return result == null ? Collections.<TextFormatParseInfoTree>emptyList() : result; + } + + /** + * Returns the parse info tree for the given field, which must be a message type. + * + * @param fieldDescriptor the @{link FieldDescriptor} of the desired sub message + * @param index the index of message value. + * @return the {@code ParseInfoTree} of the message value. {@code null} is returned if the field + * doesn't exist or the index is out of range. + * @throws IllegalArgumentException if index is out of range + */ + public TextFormatParseInfoTree getNestedTree(final FieldDescriptor fieldDescriptor, int index) { + return getFromList(getNestedTrees(fieldDescriptor), index, fieldDescriptor); + } + + /** + * Create a builder for a {@code ParseInfoTree}. + * + * @return the builder + */ + public static Builder builder() { + return new Builder(); + } + + private static <T> T getFromList(List<T> list, int index, FieldDescriptor fieldDescriptor) { + if (index >= list.size() || index < 0) { + throw new IllegalArgumentException(String.format("Illegal index field: %s, index %d", + fieldDescriptor == null ? "<null>" : fieldDescriptor.getName(), index)); + } + return list.get(index); + } + + /** + * Builder for a {@link TextFormatParseInfoTree}. + */ + public static class Builder { + + private Map<FieldDescriptor, List<TextFormatParseLocation>> locationsFromField; + + // Defines a mapping between a field's descriptor to a list of ParseInfoTrees builders for + // sub message location information. + private Map<FieldDescriptor, List<Builder>> subtreeBuildersFromField; + + /** + * Create a root level {@ParseInfoTree} builder. + */ + private Builder() { + locationsFromField = new HashMap<FieldDescriptor, List<TextFormatParseLocation>>(); + subtreeBuildersFromField = new HashMap<FieldDescriptor, List<Builder>>(); + } + + /** + * Record the starting location of a single value for a field. + * + * @param fieldDescriptor the field + * @param location source code location information + */ + public Builder setLocation( + final FieldDescriptor fieldDescriptor, TextFormatParseLocation location) { + List<TextFormatParseLocation> fieldLocations = locationsFromField.get(fieldDescriptor); + if (fieldLocations == null) { + fieldLocations = new ArrayList<TextFormatParseLocation>(); + locationsFromField.put(fieldDescriptor, fieldLocations); + } + fieldLocations.add(location); + return this; + } + + /** + * Set for a sub message. + * + * <p>A new builder is created for a sub message. The builder that is returned is a new builder. + * The return is <emph>not</emph> the invoked {@code builder.getBuilderForSubMessageField}. + * + * @param fieldDescriptor the field whose value is the submessage + * @return a new Builder for the sub message + */ + public Builder getBuilderForSubMessageField(final FieldDescriptor fieldDescriptor) { + List<Builder> submessageBuilders = subtreeBuildersFromField.get(fieldDescriptor); + if (submessageBuilders == null) { + submessageBuilders = new ArrayList<Builder>(); + subtreeBuildersFromField.put(fieldDescriptor, submessageBuilders); + } + Builder subtreeBuilder = new Builder(); + submessageBuilders.add(subtreeBuilder); + return subtreeBuilder; + } + + /** + * Build the {@code TextFormatParseInfoTree}. + * + * @return the {@code TextFormatParseInfoTree} + */ + public TextFormatParseInfoTree build() { + return new TextFormatParseInfoTree(locationsFromField, subtreeBuildersFromField); + } + } +} diff --git a/java/core/src/main/java/com/google/protobuf/TextFormatParseLocation.java b/java/core/src/main/java/com/google/protobuf/TextFormatParseLocation.java new file mode 100644 index 00000000..cce286e1 --- /dev/null +++ b/java/core/src/main/java/com/google/protobuf/TextFormatParseLocation.java @@ -0,0 +1,104 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import java.util.Arrays; + +/** + * A location in the source code. + * + * <p>A location is the starting line number and starting column number. + */ +public final class TextFormatParseLocation { + + /** + * The empty location. + */ + public static final TextFormatParseLocation EMPTY = new TextFormatParseLocation(-1, -1); + + /** + * Create a location. + * + * @param line the starting line number + * @param column the starting column number + * @return a {@code ParseLocation} + */ + static TextFormatParseLocation create(int line, int column) { + if (line == -1 && column == -1) { + return EMPTY; + } + if (line < 0 || column < 0) { + throw new IllegalArgumentException( + String.format("line and column values must be >= 0: line %d, column: %d", line, column)); + } + return new TextFormatParseLocation(line, column); + } + + private final int line; + private final int column; + + private TextFormatParseLocation(int line, int column) { + this.line = line; + this.column = column; + } + + public int getLine() { + return line; + } + + public int getColumn() { + return column; + } + + @Override + public String toString() { + return String.format("ParseLocation{line=%d, column=%d}", line, column); + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (!(o instanceof TextFormatParseLocation)) { + return false; + } + TextFormatParseLocation that = (TextFormatParseLocation) o; + return (this.line == that.getLine()) + && (this.column == that.getColumn()); + } + + @Override + public int hashCode() { + int[] values = {line, column}; + return Arrays.hashCode(values); + } +} diff --git a/java/core/src/main/java/com/google/protobuf/UnknownFieldSetLite.java b/java/core/src/main/java/com/google/protobuf/UnknownFieldSetLite.java index 435ad4d4..9500f905 100644 --- a/java/core/src/main/java/com/google/protobuf/UnknownFieldSetLite.java +++ b/java/core/src/main/java/com/google/protobuf/UnknownFieldSetLite.java @@ -61,15 +61,6 @@ public final class UnknownFieldSetLite { public static UnknownFieldSetLite getDefaultInstance() { return DEFAULT_INSTANCE; } - - /** - * Returns an empty {@code UnknownFieldSetLite.Builder}. - * - * <p>For use by generated code only. - */ - public static Builder newBuilder() { - return new Builder(); - } /** * Returns a new mutable instance. @@ -262,6 +253,21 @@ public final class UnknownFieldSetLite { return hashCode; } + /** + * Prints a String representation of the unknown field set. + * + * <p>For use by generated code only. + * + * @param buffer the buffer to write to + * @param indent the number of spaces the fields should be indented by + */ + final void printWithIndent(StringBuilder buffer, int indent) { + for (int i = 0; i < count; i++) { + int fieldNumber = WireFormat.getTagFieldNumber(tags[i]); + MessageLiteToString.printField(buffer, indent, String.valueOf(fieldNumber), objects[i]); + } + } + private void storeField(int tag, Object value) { ensureCapacity(); @@ -369,90 +375,4 @@ public final class UnknownFieldSetLite { } return this; } - - /** - * Builder for {@link UnknownFieldSetLite}s. - * - * <p>Use {@link UnknownFieldSet#newBuilder()} to construct a {@code Builder}. - * - * <p>For use by generated code only. - */ - // TODO(dweis): Update the mutable API to no longer need this builder and delete. - public static final class Builder { - - private UnknownFieldSetLite set; - - private Builder() { - this.set = null; - } - - /** - * Ensures internal state is initialized for use. - */ - private void ensureNotBuilt() { - if (set == null) { - set = new UnknownFieldSetLite(); - } - - set.checkMutable(); - } - - /** - * Parse a single field from {@code input} and merge it into this set. - * - * <p>For use by generated code only. - * - * @param tag The field's tag number, which was already parsed. - * @return {@code false} if the tag is an end group tag. - */ - boolean mergeFieldFrom(final int tag, final CodedInputStream input) throws IOException { - ensureNotBuilt(); - return set.mergeFieldFrom(tag, input); - } - - /** - * Convenience method for merging a new field containing a single varint - * value. This is used in particular when an unknown enum value is - * encountered. - * - * <p>For use by generated code only. - */ - Builder mergeVarintField(int fieldNumber, int value) { - ensureNotBuilt(); - set.mergeVarintField(fieldNumber, value); - return this; - } - - /** - * Convenience method for merging a length-delimited field. - * - * <p>For use by generated code only. - */ - public Builder mergeLengthDelimitedField(final int fieldNumber, final ByteString value) { - ensureNotBuilt(); - set.mergeLengthDelimitedField(fieldNumber, value); - return this; - } - - /** - * Build the {@link UnknownFieldSetLite} and return it. - * - * <p>Once {@code build()} has been called, the {@code Builder} will no - * longer be usable. Calling any method after {@code build()} will result - * in undefined behavior and can cause an - * {@code UnsupportedOperationException} to be thrown. - * - * <p>For use by generated code only. - */ - public UnknownFieldSetLite build() { - if (set == null) { - return DEFAULT_INSTANCE; - } - - set.checkMutable(); - set.makeImmutable(); - - return set; - } - } } diff --git a/java/core/src/main/java/com/google/protobuf/UnsafeByteOperations.java b/java/core/src/main/java/com/google/protobuf/UnsafeByteOperations.java index f443ee39..0fbf4d40 100644 --- a/java/core/src/main/java/com/google/protobuf/UnsafeByteOperations.java +++ b/java/core/src/main/java/com/google/protobuf/UnsafeByteOperations.java @@ -30,6 +30,7 @@ package com.google.protobuf; +import java.io.IOException; import java.nio.ByteBuffer; /** @@ -49,8 +50,8 @@ public final class UnsafeByteOperations { /** * An unsafe operation that returns a {@link ByteString} that is backed by the provided buffer. * - * @param buffer the Java NIO buffer to be wrapped. - * @return a {@link ByteString} backed by the provided buffer. + * @param buffer the Java NIO buffer to be wrapped + * @return a {@link ByteString} backed by the provided buffer */ public static ByteString unsafeWrap(ByteBuffer buffer) { if (buffer.hasArray()) { @@ -60,4 +61,24 @@ public final class UnsafeByteOperations { return new NioByteString(buffer); } } + + /** + * Writes the given {@link ByteString} to the provided {@link ByteOutput}. Calling this method may + * result in multiple operations on the target {@link ByteOutput} + * (i.e. for roped {@link ByteString}s). + * + * <p>This method exposes the internal backing buffer(s) of the {@link ByteString} to the {@link + * ByteOutput} in order to avoid additional copying overhead. It would be possible for a malicious + * {@link ByteOutput} to corrupt the {@link ByteString}. Use with caution! + * + * <p> NOTE: The {@link ByteOutput} <strong>MUST NOT</strong> modify the provided buffers. Doing + * so may result in corrupted data, which would be difficult to debug. + * + * @param bytes the {@link ByteString} to be written + * @param output the output to receive the bytes + * @throws IOException if an I/O error occurs + */ + public static void unsafeWriteTo(ByteString bytes, ByteOutput output) throws IOException { + bytes.writeTo(output); + } } diff --git a/java/core/src/main/java/com/google/protobuf/Utf8.java b/java/core/src/main/java/com/google/protobuf/Utf8.java index 48c7e9e6..308c69e9 100644 --- a/java/core/src/main/java/com/google/protobuf/Utf8.java +++ b/java/core/src/main/java/com/google/protobuf/Utf8.java @@ -30,6 +30,19 @@ package com.google.protobuf; +import static java.lang.Character.MAX_SURROGATE; +import static java.lang.Character.MIN_SURROGATE; +import static java.lang.Character.isSurrogatePair; +import static java.lang.Character.toCodePoint; + +import java.lang.reflect.Field; +import java.nio.Buffer; +import java.nio.ByteBuffer; +import java.security.AccessController; +import java.security.PrivilegedExceptionAction; +import java.util.logging.Level; +import java.util.logging.Logger; + /** * A set of low-level, high-performance static utility methods related * to the UTF-8 character encoding. This class has no dependencies @@ -64,9 +77,24 @@ package com.google.protobuf; * * @author martinrb@google.com (Martin Buchholz) */ +// TODO(nathanmittler): Copy changes in this class back to Guava final class Utf8 { - private Utf8() {} - + private static final Logger logger = Logger.getLogger(Utf8.class.getName()); + + /** + * UTF-8 is a runtime hot spot so we attempt to provide heavily optimized implementations + * depending on what is available on the platform. The processor is the platform-optimized + * delegate for which all methods are delegated directly to. + */ + private static final Processor processor = + UnsafeProcessor.isAvailable() ? new UnsafeProcessor() : new SafeProcessor(); + + /** + * A mask used when performing unsafe reads to determine if a long value contains any non-ASCII + * characters (i.e. any byte >= 0x80). + */ + private static final long ASCII_MASK_LONG = 0x8080808080808080L; + /** * Maximum number of bytes per Java UTF-16 char in UTF-8. * @see java.nio.charset.CharsetEncoder#maxBytesPerChar() @@ -85,6 +113,18 @@ final class Utf8 { */ public static final int MALFORMED = -1; + /** + * Used by {@code Unsafe} UTF-8 string validation logic to determine the minimum string length + * above which to employ an optimized algorithm for counting ASCII characters. The reason for this + * threshold is that for small strings, the optimization may not be beneficial or may even + * negatively impact performance since it requires additional logic to avoid unaligned reads + * (when calling {@code Unsafe.getLong}). This threshold guarantees that even if the initial + * offset is unaligned, we're guaranteed to make at least one call to {@code Unsafe.getLong()} + * which provides a performance improvement that entirely subsumes the cost of the additional + * logic. + */ + private static final int UNSAFE_COUNT_ASCII_THRESHOLD = 16; + // Other state values include the partial bytes of the incomplete // character to be decoded in the simplest way: we pack the bytes // into the state int in little-endian order. For example: @@ -112,7 +152,7 @@ final class Utf8 { * isValidUtf8(bytes, 0, bytes.length)}. */ public static boolean isValidUtf8(byte[] bytes) { - return isValidUtf8(bytes, 0, bytes.length); + return processor.isValidUtf8(bytes, 0, bytes.length); } /** @@ -125,7 +165,7 @@ final class Utf8 { * partialIsValidUtf8(bytes, index, limit) == Utf8.COMPLETE}. */ public static boolean isValidUtf8(byte[] bytes, int index, int limit) { - return partialIsValidUtf8(bytes, index, limit) == COMPLETE; + return processor.isValidUtf8(bytes, index, limit); } /** @@ -146,183 +186,8 @@ final class Utf8 { * decode the character when passed to a subsequent invocation of a * partial decoding method. */ - public static int partialIsValidUtf8( - int state, byte[] bytes, int index, int limit) { - if (state != COMPLETE) { - // The previous decoding operation was incomplete (or malformed). - // We look for a well-formed sequence consisting of bytes from - // the previous decoding operation (stored in state) together - // with bytes from the array slice. - // - // We expect such "straddler characters" to be rare. - - if (index >= limit) { // No bytes? No progress. - return state; - } - int byte1 = (byte) state; - // byte1 is never ASCII. - if (byte1 < (byte) 0xE0) { - // two-byte form - - // Simultaneously checks for illegal trailing-byte in - // leading position and overlong 2-byte form. - if (byte1 < (byte) 0xC2 || - // byte2 trailing-byte test - bytes[index++] > (byte) 0xBF) { - return MALFORMED; - } - } else if (byte1 < (byte) 0xF0) { - // three-byte form - - // Get byte2 from saved state or array - int byte2 = (byte) ~(state >> 8); - if (byte2 == 0) { - byte2 = bytes[index++]; - if (index >= limit) { - return incompleteStateFor(byte1, byte2); - } - } - if (byte2 > (byte) 0xBF || - // overlong? 5 most significant bits must not all be zero - (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) || - // illegal surrogate codepoint? - (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) || - // byte3 trailing-byte test - bytes[index++] > (byte) 0xBF) { - return MALFORMED; - } - } else { - // four-byte form - - // Get byte2 and byte3 from saved state or array - int byte2 = (byte) ~(state >> 8); - int byte3 = 0; - if (byte2 == 0) { - byte2 = bytes[index++]; - if (index >= limit) { - return incompleteStateFor(byte1, byte2); - } - } else { - byte3 = (byte) (state >> 16); - } - if (byte3 == 0) { - byte3 = bytes[index++]; - if (index >= limit) { - return incompleteStateFor(byte1, byte2, byte3); - } - } - - // If we were called with state == MALFORMED, then byte1 is 0xFF, - // which never occurs in well-formed UTF-8, and so we will return - // MALFORMED again below. - - if (byte2 > (byte) 0xBF || - // Check that 1 <= plane <= 16. Tricky optimized form of: - // if (byte1 > (byte) 0xF4 || - // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || - // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) - (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 || - // byte3 trailing-byte test - byte3 > (byte) 0xBF || - // byte4 trailing-byte test - bytes[index++] > (byte) 0xBF) { - return MALFORMED; - } - } - } - - return partialIsValidUtf8(bytes, index, limit); - } - - /** - * Tells whether the given byte array slice is a well-formed, - * malformed, or incomplete UTF-8 byte sequence. The range of bytes - * to be checked extends from index {@code index}, inclusive, to - * {@code limit}, exclusive. - * - * <p>This is a convenience method, equivalent to a call to {@code - * partialIsValidUtf8(Utf8.COMPLETE, bytes, index, limit)}. - * - * @return {@link #MALFORMED} if the partial byte sequence is - * definitely not well-formed, {@link #COMPLETE} if it is well-formed - * (no additional input needed), or if the byte sequence is - * "incomplete", i.e. apparently terminated in the middle of a character, - * an opaque integer "state" value containing enough information to - * decode the character when passed to a subsequent invocation of a - * partial decoding method. - */ - public static int partialIsValidUtf8( - byte[] bytes, int index, int limit) { - // Optimize for 100% ASCII. - // Hotspot loves small simple top-level loops like this. - while (index < limit && bytes[index] >= 0) { - index++; - } - - return (index >= limit) ? COMPLETE : - partialIsValidUtf8NonAscii(bytes, index, limit); - } - - private static int partialIsValidUtf8NonAscii( - byte[] bytes, int index, int limit) { - for (;;) { - int byte1, byte2; - - // Optimize for interior runs of ASCII bytes. - do { - if (index >= limit) { - return COMPLETE; - } - } while ((byte1 = bytes[index++]) >= 0); - - if (byte1 < (byte) 0xE0) { - // two-byte form - - if (index >= limit) { - return byte1; - } - - // Simultaneously checks for illegal trailing-byte in - // leading position and overlong 2-byte form. - if (byte1 < (byte) 0xC2 || - bytes[index++] > (byte) 0xBF) { - return MALFORMED; - } - } else if (byte1 < (byte) 0xF0) { - // three-byte form - - if (index >= limit - 1) { // incomplete sequence - return incompleteStateFor(bytes, index, limit); - } - if ((byte2 = bytes[index++]) > (byte) 0xBF || - // overlong? 5 most significant bits must not all be zero - (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) || - // check for illegal surrogate codepoints - (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) || - // byte3 trailing-byte test - bytes[index++] > (byte) 0xBF) { - return MALFORMED; - } - } else { - // four-byte form - - if (index >= limit - 2) { // incomplete sequence - return incompleteStateFor(bytes, index, limit); - } - if ((byte2 = bytes[index++]) > (byte) 0xBF || - // Check that 1 <= plane <= 16. Tricky optimized form of: - // if (byte1 > (byte) 0xF4 || - // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || - // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) - (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 || - // byte3 trailing-byte test - bytes[index++] > (byte) 0xBF || - // byte4 trailing-byte test - bytes[index++] > (byte) 0xBF) { - return MALFORMED; - } - } - } + public static int partialIsValidUtf8(int state, byte[] bytes, int index, int limit) { + return processor.partialIsValidUtf8(state, bytes, index, limit); } private static int incompleteStateFor(int byte1) { @@ -352,19 +217,31 @@ final class Utf8 { default: throw new AssertionError(); } } - + + private static int incompleteStateFor( + final ByteBuffer buffer, final int byte1, final int index, final int remaining) { + switch (remaining) { + case 0: + return incompleteStateFor(byte1); + case 1: + return incompleteStateFor(byte1, buffer.get(index)); + case 2: + return incompleteStateFor(byte1, buffer.get(index), buffer.get(index + 1)); + default: + throw new AssertionError(); + } + } // These UTF-8 handling methods are copied from Guava's Utf8 class with a modification to throw // a protocol buffer local exception. This exception is then caught in CodedOutputStream so it can // fallback to more lenient behavior. static class UnpairedSurrogateException extends IllegalArgumentException { - private UnpairedSurrogateException(int index, int length) { super("Unpaired surrogate at index " + index + " of " + length); } } - + /** * Returns the number of bytes in the UTF-8-encoded form of {@code sequence}. For a string, * this method is equivalent to {@code string.getBytes(UTF_8).length}, but is more efficient in @@ -426,56 +303,1381 @@ final class Utf8 { return utf8Length; } - static int encode(CharSequence sequence, byte[] bytes, int offset, int length) { - int utf16Length = sequence.length(); - int j = offset; - int i = 0; - int limit = offset + length; - // Designed to take advantage of - // https://wikis.oracle.com/display/HotSpotInternals/RangeCheckElimination - for (char c; i < utf16Length && i + j < limit && (c = sequence.charAt(i)) < 0x80; i++) { - bytes[j + i] = (byte) c; - } - if (i == utf16Length) { - return j + utf16Length; - } - j += i; - for (char c; i < utf16Length; i++) { - c = sequence.charAt(i); - if (c < 0x80 && j < limit) { - bytes[j++] = (byte) c; - } else if (c < 0x800 && j <= limit - 2) { // 11 bits, two UTF-8 bytes - bytes[j++] = (byte) ((0xF << 6) | (c >>> 6)); - bytes[j++] = (byte) (0x80 | (0x3F & c)); - } else if ((c < Character.MIN_SURROGATE || Character.MAX_SURROGATE < c) && j <= limit - 3) { - // Maximum single-char code point is 0xFFFF, 16 bits, three UTF-8 bytes - bytes[j++] = (byte) ((0xF << 5) | (c >>> 12)); - bytes[j++] = (byte) (0x80 | (0x3F & (c >>> 6))); - bytes[j++] = (byte) (0x80 | (0x3F & c)); - } else if (j <= limit - 4) { - // Minimum code point represented by a surrogate pair is 0x10000, 17 bits, four UTF-8 bytes - final char low; - if (i + 1 == sequence.length() - || !Character.isSurrogatePair(c, (low = sequence.charAt(++i)))) { - throw new UnpairedSurrogateException((i - 1), utf16Length); - } - int codePoint = Character.toCodePoint(c, low); - bytes[j++] = (byte) ((0xF << 4) | (codePoint >>> 18)); - bytes[j++] = (byte) (0x80 | (0x3F & (codePoint >>> 12))); - bytes[j++] = (byte) (0x80 | (0x3F & (codePoint >>> 6))); - bytes[j++] = (byte) (0x80 | (0x3F & codePoint)); + static int encode(CharSequence in, byte[] out, int offset, int length) { + return processor.encodeUtf8(in, out, offset, length); + } + // End Guava UTF-8 methods. + + /** + * Determines if the given {@link ByteBuffer} is a valid UTF-8 string. + * + * <p>Selects an optimal algorithm based on the type of {@link ByteBuffer} (i.e. heap or direct) + * and the capabilities of the platform. + * + * @param buffer the buffer to check. + * @see Utf8#isValidUtf8(byte[], int, int) + */ + static boolean isValidUtf8(ByteBuffer buffer) { + return processor.isValidUtf8(buffer, buffer.position(), buffer.remaining()); + } + + /** + * Determines if the given {@link ByteBuffer} is a partially valid UTF-8 string. + * + * <p>Selects an optimal algorithm based on the type of {@link ByteBuffer} (i.e. heap or direct) + * and the capabilities of the platform. + * + * @param buffer the buffer to check. + * @see Utf8#partialIsValidUtf8(int, byte[], int, int) + */ + static int partialIsValidUtf8(int state, ByteBuffer buffer, int index, int limit) { + return processor.partialIsValidUtf8(state, buffer, index, limit); + } + + /** + * Encodes the given characters to the target {@link ByteBuffer} using UTF-8 encoding. + * + * <p>Selects an optimal algorithm based on the type of {@link ByteBuffer} (i.e. heap or direct) + * and the capabilities of the platform. + * + * @param in the source string to be encoded + * @param out the target buffer to receive the encoded string. + * @see Utf8#encode(CharSequence, byte[], int, int) + */ + static void encodeUtf8(CharSequence in, ByteBuffer out) { + processor.encodeUtf8(in, out); + } + + /** + * Counts (approximately) the number of consecutive ASCII characters in the given buffer. + * The byte order of the {@link ByteBuffer} does not matter, so performance can be improved if + * native byte order is used (i.e. no byte-swapping in {@link ByteBuffer#getLong(int)}). + * + * @param buffer the buffer to be scanned for ASCII chars + * @param index the starting index of the scan + * @param limit the limit within buffer for the scan + * @return the number of ASCII characters found. The stopping position will be at or + * before the first non-ASCII byte. + */ + private static int estimateConsecutiveAscii(ByteBuffer buffer, int index, int limit) { + int i = index; + final int lim = limit - 7; + // This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII). + // To speed things up further, we're reading longs instead of bytes so we use a mask to + // determine if any byte in the current long is non-ASCII. + for (; i < lim && (buffer.getLong(i) & ASCII_MASK_LONG) == 0; i += 8) {} + return i - index; + } + + /** + * A processor of UTF-8 strings, providing methods for checking validity and encoding. + */ + // TODO(nathanmittler): Add support for Memory/MemoryBlock on Android. + abstract static class Processor { + /** + * Returns {@code true} if the given byte array slice is a + * well-formed UTF-8 byte sequence. The range of bytes to be + * checked extends from index {@code index}, inclusive, to {@code + * limit}, exclusive. + * + * <p>This is a convenience method, equivalent to {@code + * partialIsValidUtf8(bytes, index, limit) == Utf8.COMPLETE}. + */ + final boolean isValidUtf8(byte[] bytes, int index, int limit) { + return partialIsValidUtf8(COMPLETE, bytes, index, limit) == COMPLETE; + } + + /** + * Tells whether the given byte array slice is a well-formed, + * malformed, or incomplete UTF-8 byte sequence. The range of bytes + * to be checked extends from index {@code index}, inclusive, to + * {@code limit}, exclusive. + * + * @param state either {@link Utf8#COMPLETE} (if this is the initial decoding + * operation) or the value returned from a call to a partial decoding method + * for the previous bytes + * + * @return {@link #MALFORMED} if the partial byte sequence is + * definitely not well-formed, {@link #COMPLETE} if it is well-formed + * (no additional input needed), or if the byte sequence is + * "incomplete", i.e. apparently terminated in the middle of a character, + * an opaque integer "state" value containing enough information to + * decode the character when passed to a subsequent invocation of a + * partial decoding method. + */ + abstract int partialIsValidUtf8(int state, byte[] bytes, int index, int limit); + + /** + * Returns {@code true} if the given portion of the {@link ByteBuffer} is a + * well-formed UTF-8 byte sequence. The range of bytes to be + * checked extends from index {@code index}, inclusive, to {@code + * limit}, exclusive. + * + * <p>This is a convenience method, equivalent to {@code + * partialIsValidUtf8(bytes, index, limit) == Utf8.COMPLETE}. + */ + final boolean isValidUtf8(ByteBuffer buffer, int index, int limit) { + return partialIsValidUtf8(COMPLETE, buffer, index, limit) == COMPLETE; + } + + /** + * Indicates whether or not the given buffer contains a valid UTF-8 string. + * + * @param buffer the buffer to check. + * @return {@code true} if the given buffer contains a valid UTF-8 string. + */ + final int partialIsValidUtf8( + final int state, final ByteBuffer buffer, int index, final int limit) { + if (buffer.hasArray()) { + final int offset = buffer.arrayOffset(); + return partialIsValidUtf8(state, buffer.array(), offset + index, offset + limit); + } else if (buffer.isDirect()){ + return partialIsValidUtf8Direct(state, buffer, index, limit); + } + return partialIsValidUtf8Default(state, buffer, index, limit); + } + + /** + * Performs validation for direct {@link ByteBuffer} instances. + */ + abstract int partialIsValidUtf8Direct( + final int state, final ByteBuffer buffer, int index, final int limit); + + /** + * Performs validation for {@link ByteBuffer} instances using the {@link ByteBuffer} API rather + * than potentially faster approaches. This first completes validation for the current + * character (provided by {@code state}) and then finishes validation for the sequence. + */ + final int partialIsValidUtf8Default( + final int state, final ByteBuffer buffer, int index, final int limit) { + if (state != COMPLETE) { + // The previous decoding operation was incomplete (or malformed). + // We look for a well-formed sequence consisting of bytes from + // the previous decoding operation (stored in state) together + // with bytes from the array slice. + // + // We expect such "straddler characters" to be rare. + + if (index >= limit) { // No bytes? No progress. + return state; + } + + byte byte1 = (byte) state; + // byte1 is never ASCII. + if (byte1 < (byte) 0xE0) { + // two-byte form + + // Simultaneously checks for illegal trailing-byte in + // leading position and overlong 2-byte form. + if (byte1 < (byte) 0xC2 + // byte2 trailing-byte test + || buffer.get(index++) > (byte) 0xBF) { + return MALFORMED; + } + } else if (byte1 < (byte) 0xF0) { + // three-byte form + + // Get byte2 from saved state or array + byte byte2 = (byte) ~(state >> 8); + if (byte2 == 0) { + byte2 = buffer.get(index++); + if (index >= limit) { + return incompleteStateFor(byte1, byte2); + } + } + if (byte2 > (byte) 0xBF + // overlong? 5 most significant bits must not all be zero + || (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) + // illegal surrogate codepoint? + || (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) + // byte3 trailing-byte test + || buffer.get(index++) > (byte) 0xBF) { + return MALFORMED; + } + } else { + // four-byte form + + // Get byte2 and byte3 from saved state or array + byte byte2 = (byte) ~(state >> 8); + byte byte3 = 0; + if (byte2 == 0) { + byte2 = buffer.get(index++); + if (index >= limit) { + return incompleteStateFor(byte1, byte2); + } + } else { + byte3 = (byte) (state >> 16); + } + if (byte3 == 0) { + byte3 = buffer.get(index++); + if (index >= limit) { + return incompleteStateFor(byte1, byte2, byte3); + } + } + + // If we were called with state == MALFORMED, then byte1 is 0xFF, + // which never occurs in well-formed UTF-8, and so we will return + // MALFORMED again below. + + if (byte2 > (byte) 0xBF + // Check that 1 <= plane <= 16. Tricky optimized form of: + // if (byte1 > (byte) 0xF4 || + // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || + // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) + || (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 + // byte3 trailing-byte test + || byte3 > (byte) 0xBF + // byte4 trailing-byte test + || buffer.get(index++) > (byte) 0xBF) { + return MALFORMED; + } + } + } + + // Finish validation for the sequence. + return partialIsValidUtf8(buffer, index, limit); + } + + /** + * Performs validation for {@link ByteBuffer} instances using the {@link ByteBuffer} API rather + * than potentially faster approaches. + */ + private static int partialIsValidUtf8(final ByteBuffer buffer, int index, final int limit) { + index += estimateConsecutiveAscii(buffer, index, limit); + + for (;;) { + // Optimize for interior runs of ASCII bytes. + // TODO(nathanmittler): Consider checking 8 bytes at a time after some threshold? + // Maybe after seeing a few in a row that are ASCII, go back to fast mode? + int byte1; + do { + if (index >= limit) { + return COMPLETE; + } + } while ((byte1 = buffer.get(index++)) >= 0); + + // If we're here byte1 is not ASCII. Only need to handle 2-4 byte forms. + if (byte1 < (byte) 0xE0) { + // Two-byte form (110xxxxx 10xxxxxx) + if (index >= limit) { + // Incomplete sequence + return byte1; + } + + // Simultaneously checks for illegal trailing-byte in + // leading position and overlong 2-byte form. + if (byte1 < (byte) 0xC2 || buffer.get(index) > (byte) 0xBF) { + return MALFORMED; + } + index++; + } else if (byte1 < (byte) 0xF0) { + // Three-byte form (1110xxxx 10xxxxxx 10xxxxxx) + if (index >= limit - 1) { + // Incomplete sequence + return incompleteStateFor(buffer, byte1, index, limit - index); + } + + final byte byte2 = buffer.get(index++); + if (byte2 > (byte) 0xBF + // overlong? 5 most significant bits must not all be zero + || (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) + // check for illegal surrogate codepoints + || (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) + // byte3 trailing-byte test + || buffer.get(index) > (byte) 0xBF) { + return MALFORMED; + } + index++; + } else { + // Four-byte form (1110xxxx 10xxxxxx 10xxxxxx 10xxxxxx) + if (index >= limit - 2) { + // Incomplete sequence + return incompleteStateFor(buffer, byte1, index, limit - index); + } + + // TODO(nathanmittler): Consider using getInt() to improve performance. + final int byte2 = buffer.get(index++); + if (byte2 > (byte) 0xBF + // Check that 1 <= plane <= 16. Tricky optimized form of: + // if (byte1 > (byte) 0xF4 || + // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || + // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) + || (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 + // byte3 trailing-byte test + || buffer.get(index++) > (byte) 0xBF + // byte4 trailing-byte test + || buffer.get(index++) > (byte) 0xBF) { + return MALFORMED; + } + } + } + } + + /** + * Encodes an input character sequence ({@code in}) to UTF-8 in the target array ({@code out}). + * For a string, this method is similar to + * <pre>{@code + * byte[] a = string.getBytes(UTF_8); + * System.arraycopy(a, 0, bytes, offset, a.length); + * return offset + a.length; + * }</pre> + * + * but is more efficient in both time and space. One key difference is that this method + * requires paired surrogates, and therefore does not support chunking. + * While {@code String.getBytes(UTF_8)} replaces unpaired surrogates with the default + * replacement character, this method throws {@link UnpairedSurrogateException}. + * + * <p>To ensure sufficient space in the output buffer, either call {@link #encodedLength} to + * compute the exact amount needed, or leave room for + * {@code Utf8.MAX_BYTES_PER_CHAR * sequence.length()}, which is the largest possible number + * of bytes that any input can be encoded to. + * + * @param in the input character sequence to be encoded + * @param out the target array + * @param offset the starting offset in {@code bytes} to start writing at + * @param length the length of the {@code bytes}, starting from {@code offset} + * @throws UnpairedSurrogateException if {@code sequence} contains ill-formed UTF-16 (unpaired + * surrogates) + * @throws ArrayIndexOutOfBoundsException if {@code sequence} encoded in UTF-8 is longer than + * {@code bytes.length - offset} + * @return the new offset, equivalent to {@code offset + Utf8.encodedLength(sequence)} + */ + abstract int encodeUtf8(CharSequence in, byte[] out, int offset, int length); + + /** + * Encodes an input character sequence ({@code in}) to UTF-8 in the target buffer ({@code out}). + * Upon returning from this method, the {@code out} position will point to the position after + * the last encoded byte. This method requires paired surrogates, and therefore does not + * support chunking. + * + * <p>To ensure sufficient space in the output buffer, either call {@link #encodedLength} to + * compute the exact amount needed, or leave room for + * {@code Utf8.MAX_BYTES_PER_CHAR * in.length()}, which is the largest possible number + * of bytes that any input can be encoded to. + * + * @param in the source character sequence to be encoded + * @param out the target buffer + * @throws UnpairedSurrogateException if {@code in} contains ill-formed UTF-16 (unpaired + * surrogates) + * @throws ArrayIndexOutOfBoundsException if {@code in} encoded in UTF-8 is longer than + * {@code out.remaining()} + */ + final void encodeUtf8(CharSequence in, ByteBuffer out) { + if (out.hasArray()) { + final int offset = out.arrayOffset(); + int endIndex = + Utf8.encode(in, out.array(), offset + out.position(), out.remaining()); + out.position(endIndex - offset); + } else if (out.isDirect()) { + encodeUtf8Direct(in, out); } else { - // If we are surrogates and we're not a surrogate pair, always throw an - // IllegalArgumentException instead of an ArrayOutOfBoundsException. - if ((Character.MIN_SURROGATE <= c && c <= Character.MAX_SURROGATE) - && (i + 1 == sequence.length() - || !Character.isSurrogatePair(c, sequence.charAt(i + 1)))) { - throw new UnpairedSurrogateException(i, utf16Length); + encodeUtf8Default(in, out); + } + } + + /** + * Encodes the input character sequence to a direct {@link ByteBuffer} instance. + */ + abstract void encodeUtf8Direct(CharSequence in, ByteBuffer out); + + /** + * Encodes the input character sequence to a {@link ByteBuffer} instance using the {@link + * ByteBuffer} API, rather than potentially faster approaches. + */ + final void encodeUtf8Default(CharSequence in, ByteBuffer out) { + final int inLength = in.length(); + int outIx = out.position(); + int inIx = 0; + + // Since ByteBuffer.putXXX() already checks boundaries for us, no need to explicitly check + // access. Assume the buffer is big enough and let it handle the out of bounds exception + // if it occurs. + try { + // Designed to take advantage of + // https://wikis.oracle.com/display/HotSpotInternals/RangeCheckElimination + for (char c; inIx < inLength && (c = in.charAt(inIx)) < 0x80; ++inIx) { + out.put(outIx + inIx, (byte) c); } - throw new ArrayIndexOutOfBoundsException("Failed writing " + c + " at index " + j); + if (inIx == inLength) { + // Successfully encoded the entire string. + out.position(outIx + inIx); + return; + } + + outIx += inIx; + for (char c; inIx < inLength; ++inIx, ++outIx) { + c = in.charAt(inIx); + if (c < 0x80) { + // One byte (0xxx xxxx) + out.put(outIx, (byte) c); + } else if (c < 0x800) { + // Two bytes (110x xxxx 10xx xxxx) + + // Benchmarks show put performs better than putShort here (for HotSpot). + out.put(outIx++, (byte) (0xC0 | (c >>> 6))); + out.put(outIx, (byte) (0x80 | (0x3F & c))); + } else if (c < MIN_SURROGATE || MAX_SURROGATE < c) { + // Three bytes (1110 xxxx 10xx xxxx 10xx xxxx) + // Maximum single-char code point is 0xFFFF, 16 bits. + + // Benchmarks show put performs better than putShort here (for HotSpot). + out.put(outIx++, (byte) (0xE0 | (c >>> 12))); + out.put(outIx++, (byte) (0x80 | (0x3F & (c >>> 6)))); + out.put(outIx, (byte) (0x80 | (0x3F & c))); + } else { + // Four bytes (1111 xxxx 10xx xxxx 10xx xxxx 10xx xxxx) + + // Minimum code point represented by a surrogate pair is 0x10000, 17 bits, four UTF-8 + // bytes + final char low; + if (inIx + 1 == inLength || !isSurrogatePair(c, (low = in.charAt(++inIx)))) { + throw new UnpairedSurrogateException(inIx, inLength); + } + // TODO(nathanmittler): Consider using putInt() to improve performance. + int codePoint = toCodePoint(c, low); + out.put(outIx++, (byte) ((0xF << 4) | (codePoint >>> 18))); + out.put(outIx++, (byte) (0x80 | (0x3F & (codePoint >>> 12)))); + out.put(outIx++, (byte) (0x80 | (0x3F & (codePoint >>> 6)))); + out.put(outIx, (byte) (0x80 | (0x3F & codePoint))); + } + } + + // Successfully encoded the entire string. + out.position(outIx); + } catch (IndexOutOfBoundsException e) { + // TODO(nathanmittler): Consider making the API throw IndexOutOfBoundsException instead. + + // If we failed in the outer ASCII loop, outIx will not have been updated. In this case, + // use inIx to determine the bad write index. + int badWriteIndex = out.position() + Math.max(inIx, outIx - out.position() + 1); + throw new ArrayIndexOutOfBoundsException( + "Failed writing " + in.charAt(inIx) + " at index " + badWriteIndex); } } - return j; } - // End Guava UTF-8 methods. + + /** + * {@link Processor} implementation that does not use any {@code sun.misc.Unsafe} methods. + */ + static final class SafeProcessor extends Processor { + @Override + int partialIsValidUtf8(int state, byte[] bytes, int index, int limit) { + if (state != COMPLETE) { + // The previous decoding operation was incomplete (or malformed). + // We look for a well-formed sequence consisting of bytes from + // the previous decoding operation (stored in state) together + // with bytes from the array slice. + // + // We expect such "straddler characters" to be rare. + + if (index >= limit) { // No bytes? No progress. + return state; + } + int byte1 = (byte) state; + // byte1 is never ASCII. + if (byte1 < (byte) 0xE0) { + // two-byte form + + // Simultaneously checks for illegal trailing-byte in + // leading position and overlong 2-byte form. + if (byte1 < (byte) 0xC2 + // byte2 trailing-byte test + || bytes[index++] > (byte) 0xBF) { + return MALFORMED; + } + } else if (byte1 < (byte) 0xF0) { + // three-byte form + + // Get byte2 from saved state or array + int byte2 = (byte) ~(state >> 8); + if (byte2 == 0) { + byte2 = bytes[index++]; + if (index >= limit) { + return incompleteStateFor(byte1, byte2); + } + } + if (byte2 > (byte) 0xBF + // overlong? 5 most significant bits must not all be zero + || (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) + // illegal surrogate codepoint? + || (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) + // byte3 trailing-byte test + || bytes[index++] > (byte) 0xBF) { + return MALFORMED; + } + } else { + // four-byte form + + // Get byte2 and byte3 from saved state or array + int byte2 = (byte) ~(state >> 8); + int byte3 = 0; + if (byte2 == 0) { + byte2 = bytes[index++]; + if (index >= limit) { + return incompleteStateFor(byte1, byte2); + } + } else { + byte3 = (byte) (state >> 16); + } + if (byte3 == 0) { + byte3 = bytes[index++]; + if (index >= limit) { + return incompleteStateFor(byte1, byte2, byte3); + } + } + + // If we were called with state == MALFORMED, then byte1 is 0xFF, + // which never occurs in well-formed UTF-8, and so we will return + // MALFORMED again below. + + if (byte2 > (byte) 0xBF + // Check that 1 <= plane <= 16. Tricky optimized form of: + // if (byte1 > (byte) 0xF4 || + // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || + // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) + || (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 + // byte3 trailing-byte test + || byte3 > (byte) 0xBF + // byte4 trailing-byte test + || bytes[index++] > (byte) 0xBF) { + return MALFORMED; + } + } + } + + return partialIsValidUtf8(bytes, index, limit); + } + + @Override + int partialIsValidUtf8Direct(int state, ByteBuffer buffer, int index, int limit) { + // For safe processing, we have to use the ByteBuffer API. + return partialIsValidUtf8Default(state, buffer, index, limit); + } + + @Override + int encodeUtf8(CharSequence in, byte[] out, int offset, int length) { + int utf16Length = in.length(); + int j = offset; + int i = 0; + int limit = offset + length; + // Designed to take advantage of + // https://wikis.oracle.com/display/HotSpotInternals/RangeCheckElimination + for (char c; i < utf16Length && i + j < limit && (c = in.charAt(i)) < 0x80; i++) { + out[j + i] = (byte) c; + } + if (i == utf16Length) { + return j + utf16Length; + } + j += i; + for (char c; i < utf16Length; i++) { + c = in.charAt(i); + if (c < 0x80 && j < limit) { + out[j++] = (byte) c; + } else if (c < 0x800 && j <= limit - 2) { // 11 bits, two UTF-8 bytes + out[j++] = (byte) ((0xF << 6) | (c >>> 6)); + out[j++] = (byte) (0x80 | (0x3F & c)); + } else if ((c < Character.MIN_SURROGATE || Character.MAX_SURROGATE < c) && j <= limit - 3) { + // Maximum single-char code point is 0xFFFF, 16 bits, three UTF-8 bytes + out[j++] = (byte) ((0xF << 5) | (c >>> 12)); + out[j++] = (byte) (0x80 | (0x3F & (c >>> 6))); + out[j++] = (byte) (0x80 | (0x3F & c)); + } else if (j <= limit - 4) { + // Minimum code point represented by a surrogate pair is 0x10000, 17 bits, + // four UTF-8 bytes + final char low; + if (i + 1 == in.length() + || !Character.isSurrogatePair(c, (low = in.charAt(++i)))) { + throw new UnpairedSurrogateException((i - 1), utf16Length); + } + int codePoint = Character.toCodePoint(c, low); + out[j++] = (byte) ((0xF << 4) | (codePoint >>> 18)); + out[j++] = (byte) (0x80 | (0x3F & (codePoint >>> 12))); + out[j++] = (byte) (0x80 | (0x3F & (codePoint >>> 6))); + out[j++] = (byte) (0x80 | (0x3F & codePoint)); + } else { + // If we are surrogates and we're not a surrogate pair, always throw an + // UnpairedSurrogateException instead of an ArrayOutOfBoundsException. + if ((Character.MIN_SURROGATE <= c && c <= Character.MAX_SURROGATE) + && (i + 1 == in.length() + || !Character.isSurrogatePair(c, in.charAt(i + 1)))) { + throw new UnpairedSurrogateException(i, utf16Length); + } + throw new ArrayIndexOutOfBoundsException("Failed writing " + c + " at index " + j); + } + } + return j; + } + + @Override + void encodeUtf8Direct(CharSequence in, ByteBuffer out) { + // For safe processing, we have to use the ByteBuffer API. + encodeUtf8Default(in, out); + } + + private static int partialIsValidUtf8(byte[] bytes, int index, int limit) { + // Optimize for 100% ASCII (Hotspot loves small simple top-level loops like this). + // This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII). + while (index < limit && bytes[index] >= 0) { + index++; + } + + return (index >= limit) ? COMPLETE : partialIsValidUtf8NonAscii(bytes, index, limit); + } + + private static int partialIsValidUtf8NonAscii(byte[] bytes, int index, int limit) { + for (;;) { + int byte1, byte2; + + // Optimize for interior runs of ASCII bytes. + do { + if (index >= limit) { + return COMPLETE; + } + } while ((byte1 = bytes[index++]) >= 0); + + if (byte1 < (byte) 0xE0) { + // two-byte form + + if (index >= limit) { + // Incomplete sequence + return byte1; + } + + // Simultaneously checks for illegal trailing-byte in + // leading position and overlong 2-byte form. + if (byte1 < (byte) 0xC2 + || bytes[index++] > (byte) 0xBF) { + return MALFORMED; + } + } else if (byte1 < (byte) 0xF0) { + // three-byte form + + if (index >= limit - 1) { // incomplete sequence + return incompleteStateFor(bytes, index, limit); + } + if ((byte2 = bytes[index++]) > (byte) 0xBF + // overlong? 5 most significant bits must not all be zero + || (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) + // check for illegal surrogate codepoints + || (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) + // byte3 trailing-byte test + || bytes[index++] > (byte) 0xBF) { + return MALFORMED; + } + } else { + // four-byte form + + if (index >= limit - 2) { // incomplete sequence + return incompleteStateFor(bytes, index, limit); + } + if ((byte2 = bytes[index++]) > (byte) 0xBF + // Check that 1 <= plane <= 16. Tricky optimized form of: + // if (byte1 > (byte) 0xF4 || + // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || + // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) + || (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 + // byte3 trailing-byte test + || bytes[index++] > (byte) 0xBF + // byte4 trailing-byte test + || bytes[index++] > (byte) 0xBF) { + return MALFORMED; + } + } + } + } + } + + /** + * {@link Processor} that uses {@code sun.misc.Unsafe} where possible to improve performance. + */ + static final class UnsafeProcessor extends Processor { + private static final sun.misc.Unsafe UNSAFE = getUnsafe(); + private static final long BUFFER_ADDRESS_OFFSET = + fieldOffset(field(Buffer.class, "address")); + private static final int ARRAY_BASE_OFFSET = byteArrayBaseOffset(); + + /** + * We only use Unsafe operations if we have access to direct {@link ByteBuffer}'s address + * and the array base offset is a multiple of 8 (needed by Unsafe.getLong()). + */ + private static final boolean AVAILABLE = + BUFFER_ADDRESS_OFFSET != -1 && ARRAY_BASE_OFFSET % 8 == 0; + + /** + * Indicates whether or not all required unsafe operations are supported on this platform. + */ + static boolean isAvailable() { + return AVAILABLE; + } + + @Override + int partialIsValidUtf8(int state, byte[] bytes, final int index, final int limit) { + if ((index | limit | bytes.length - limit) < 0) { + throw new ArrayIndexOutOfBoundsException( + String.format("Array length=%d, index=%d, limit=%d", bytes.length, index, limit)); + } + long offset = ARRAY_BASE_OFFSET + index; + final long offsetLimit = ARRAY_BASE_OFFSET + limit; + if (state != COMPLETE) { + // The previous decoding operation was incomplete (or malformed). + // We look for a well-formed sequence consisting of bytes from + // the previous decoding operation (stored in state) together + // with bytes from the array slice. + // + // We expect such "straddler characters" to be rare. + + if (offset >= offsetLimit) { // No bytes? No progress. + return state; + } + int byte1 = (byte) state; + // byte1 is never ASCII. + if (byte1 < (byte) 0xE0) { + // two-byte form + + // Simultaneously checks for illegal trailing-byte in + // leading position and overlong 2-byte form. + if (byte1 < (byte) 0xC2 + // byte2 trailing-byte test + || UNSAFE.getByte(bytes, offset++) > (byte) 0xBF) { + return MALFORMED; + } + } else if (byte1 < (byte) 0xF0) { + // three-byte form + + // Get byte2 from saved state or array + int byte2 = (byte) ~(state >> 8); + if (byte2 == 0) { + byte2 = UNSAFE.getByte(bytes, offset++); + if (offset >= offsetLimit) { + return incompleteStateFor(byte1, byte2); + } + } + if (byte2 > (byte) 0xBF + // overlong? 5 most significant bits must not all be zero + || (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) + // illegal surrogate codepoint? + || (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) + // byte3 trailing-byte test + || UNSAFE.getByte(bytes, offset++) > (byte) 0xBF) { + return MALFORMED; + } + } else { + // four-byte form + + // Get byte2 and byte3 from saved state or array + int byte2 = (byte) ~(state >> 8); + int byte3 = 0; + if (byte2 == 0) { + byte2 = UNSAFE.getByte(bytes, offset++); + if (offset >= offsetLimit) { + return incompleteStateFor(byte1, byte2); + } + } else { + byte3 = (byte) (state >> 16); + } + if (byte3 == 0) { + byte3 = UNSAFE.getByte(bytes, offset++); + if (offset >= offsetLimit) { + return incompleteStateFor(byte1, byte2, byte3); + } + } + + // If we were called with state == MALFORMED, then byte1 is 0xFF, + // which never occurs in well-formed UTF-8, and so we will return + // MALFORMED again below. + + if (byte2 > (byte) 0xBF + // Check that 1 <= plane <= 16. Tricky optimized form of: + // if (byte1 > (byte) 0xF4 || + // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || + // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) + || (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 + // byte3 trailing-byte test + || byte3 > (byte) 0xBF + // byte4 trailing-byte test + || UNSAFE.getByte(bytes, offset++) > (byte) 0xBF) { + return MALFORMED; + } + } + } + + return partialIsValidUtf8(bytes, offset, (int) (offsetLimit - offset)); + } + + @Override + int partialIsValidUtf8Direct( + final int state, ByteBuffer buffer, final int index, final int limit) { + if ((index | limit | buffer.limit() - limit) < 0) { + throw new ArrayIndexOutOfBoundsException( + String.format("buffer limit=%d, index=%d, limit=%d", buffer.limit(), index, limit)); + } + long address = addressOffset(buffer) + index; + final long addressLimit = address + (limit - index); + if (state != COMPLETE) { + // The previous decoding operation was incomplete (or malformed). + // We look for a well-formed sequence consisting of bytes from + // the previous decoding operation (stored in state) together + // with bytes from the array slice. + // + // We expect such "straddler characters" to be rare. + + if (address >= addressLimit) { // No bytes? No progress. + return state; + } + + final int byte1 = (byte) state; + // byte1 is never ASCII. + if (byte1 < (byte) 0xE0) { + // two-byte form + + // Simultaneously checks for illegal trailing-byte in + // leading position and overlong 2-byte form. + if (byte1 < (byte) 0xC2 + // byte2 trailing-byte test + || UNSAFE.getByte(address++) > (byte) 0xBF) { + return MALFORMED; + } + } else if (byte1 < (byte) 0xF0) { + // three-byte form + + // Get byte2 from saved state or array + int byte2 = (byte) ~(state >> 8); + if (byte2 == 0) { + byte2 = UNSAFE.getByte(address++); + if (address >= addressLimit) { + return incompleteStateFor(byte1, byte2); + } + } + if (byte2 > (byte) 0xBF + // overlong? 5 most significant bits must not all be zero + || (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) + // illegal surrogate codepoint? + || (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) + // byte3 trailing-byte test + || UNSAFE.getByte(address++) > (byte) 0xBF) { + return MALFORMED; + } + } else { + // four-byte form + + // Get byte2 and byte3 from saved state or array + int byte2 = (byte) ~(state >> 8); + int byte3 = 0; + if (byte2 == 0) { + byte2 = UNSAFE.getByte(address++); + if (address >= addressLimit) { + return incompleteStateFor(byte1, byte2); + } + } else { + byte3 = (byte) (state >> 16); + } + if (byte3 == 0) { + byte3 = UNSAFE.getByte(address++); + if (address >= addressLimit) { + return incompleteStateFor(byte1, byte2, byte3); + } + } + + // If we were called with state == MALFORMED, then byte1 is 0xFF, + // which never occurs in well-formed UTF-8, and so we will return + // MALFORMED again below. + + if (byte2 > (byte) 0xBF + // Check that 1 <= plane <= 16. Tricky optimized form of: + // if (byte1 > (byte) 0xF4 || + // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || + // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) + || (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 + // byte3 trailing-byte test + || byte3 > (byte) 0xBF + // byte4 trailing-byte test + || UNSAFE.getByte(address++) > (byte) 0xBF) { + return MALFORMED; + } + } + } + + return partialIsValidUtf8(address, (int) (addressLimit - address)); + } + + @Override + int encodeUtf8(final CharSequence in, final byte[] out, final int offset, final int length) { + long outIx = ARRAY_BASE_OFFSET + offset; + final long outLimit = outIx + length; + final int inLimit = in.length(); + if (inLimit > length || out.length - length < offset) { + // Not even enough room for an ASCII-encoded string. + throw new ArrayIndexOutOfBoundsException( + "Failed writing " + in.charAt(inLimit - 1) + " at index " + (offset + length)); + } + + // Designed to take advantage of + // https://wikis.oracle.com/display/HotSpotInternals/RangeCheckElimination + int inIx = 0; + for (char c; inIx < inLimit && (c = in.charAt(inIx)) < 0x80; ++inIx) { + UNSAFE.putByte(out, outIx++, (byte) c); + } + if (inIx == inLimit) { + // We're done, it was ASCII encoded. + return (int) (outIx - ARRAY_BASE_OFFSET); + } + + for (char c; inIx < inLimit; ++inIx) { + c = in.charAt(inIx); + if (c < 0x80 && outIx < outLimit) { + UNSAFE.putByte(out, outIx++, (byte) c); + } else if (c < 0x800 && outIx <= outLimit - 2L) { // 11 bits, two UTF-8 bytes + UNSAFE.putByte(out, outIx++, (byte) ((0xF << 6) | (c >>> 6))); + UNSAFE.putByte(out, outIx++, (byte) (0x80 | (0x3F & c))); + } else if ((c < MIN_SURROGATE || MAX_SURROGATE < c) && outIx <= outLimit - 3L) { + // Maximum single-char code point is 0xFFFF, 16 bits, three UTF-8 bytes + UNSAFE.putByte(out, outIx++, (byte) ((0xF << 5) | (c >>> 12))); + UNSAFE.putByte(out, outIx++, (byte) (0x80 | (0x3F & (c >>> 6)))); + UNSAFE.putByte(out, outIx++, (byte) (0x80 | (0x3F & c))); + } else if (outIx <= outLimit - 4L) { + // Minimum code point represented by a surrogate pair is 0x10000, 17 bits, four UTF-8 + // bytes + final char low; + if (inIx + 1 == inLimit || !isSurrogatePair(c, (low = in.charAt(++inIx)))) { + throw new UnpairedSurrogateException((inIx - 1), inLimit); + } + int codePoint = toCodePoint(c, low); + UNSAFE.putByte(out, outIx++, (byte) ((0xF << 4) | (codePoint >>> 18))); + UNSAFE.putByte(out, outIx++, (byte) (0x80 | (0x3F & (codePoint >>> 12)))); + UNSAFE.putByte(out, outIx++, (byte) (0x80 | (0x3F & (codePoint >>> 6)))); + UNSAFE.putByte(out, outIx++, (byte) (0x80 | (0x3F & codePoint))); + } else { + if ((MIN_SURROGATE <= c && c <= MAX_SURROGATE) + && (inIx + 1 == inLimit || !isSurrogatePair(c, in.charAt(inIx + 1)))) { + // We are surrogates and we're not a surrogate pair. + throw new UnpairedSurrogateException(inIx, inLimit); + } + // Not enough space in the output buffer. + throw new ArrayIndexOutOfBoundsException("Failed writing " + c + " at index " + outIx); + } + } + + // All bytes have been encoded. + return (int) (outIx - ARRAY_BASE_OFFSET); + } + + @Override + void encodeUtf8Direct(CharSequence in, ByteBuffer out) { + final long address = addressOffset(out); + long outIx = address + out.position(); + final long outLimit = address + out.limit(); + final int inLimit = in.length(); + if (inLimit > outLimit - outIx) { + // Not even enough room for an ASCII-encoded string. + throw new ArrayIndexOutOfBoundsException( + "Failed writing " + in.charAt(inLimit - 1) + " at index " + out.limit()); + } + + // Designed to take advantage of + // https://wikis.oracle.com/display/HotSpotInternals/RangeCheckElimination + int inIx = 0; + for (char c; inIx < inLimit && (c = in.charAt(inIx)) < 0x80; ++inIx) { + UNSAFE.putByte(outIx++, (byte) c); + } + if (inIx == inLimit) { + // We're done, it was ASCII encoded. + out.position((int) (outIx - address)); + return; + } + + for (char c; inIx < inLimit; ++inIx) { + c = in.charAt(inIx); + if (c < 0x80 && outIx < outLimit) { + UNSAFE.putByte(outIx++, (byte) c); + } else if (c < 0x800 && outIx <= outLimit - 2L) { // 11 bits, two UTF-8 bytes + UNSAFE.putByte(outIx++, (byte) ((0xF << 6) | (c >>> 6))); + UNSAFE.putByte(outIx++, (byte) (0x80 | (0x3F & c))); + } else if ((c < MIN_SURROGATE || MAX_SURROGATE < c) && outIx <= outLimit - 3L) { + // Maximum single-char code point is 0xFFFF, 16 bits, three UTF-8 bytes + UNSAFE.putByte(outIx++, (byte) ((0xF << 5) | (c >>> 12))); + UNSAFE.putByte(outIx++, (byte) (0x80 | (0x3F & (c >>> 6)))); + UNSAFE.putByte(outIx++, (byte) (0x80 | (0x3F & c))); + } else if (outIx <= outLimit - 4L) { + // Minimum code point represented by a surrogate pair is 0x10000, 17 bits, four UTF-8 + // bytes + final char low; + if (inIx + 1 == inLimit || !isSurrogatePair(c, (low = in.charAt(++inIx)))) { + throw new UnpairedSurrogateException((inIx - 1), inLimit); + } + int codePoint = toCodePoint(c, low); + UNSAFE.putByte(outIx++, (byte) ((0xF << 4) | (codePoint >>> 18))); + UNSAFE.putByte(outIx++, (byte) (0x80 | (0x3F & (codePoint >>> 12)))); + UNSAFE.putByte(outIx++, (byte) (0x80 | (0x3F & (codePoint >>> 6)))); + UNSAFE.putByte(outIx++, (byte) (0x80 | (0x3F & codePoint))); + } else { + if ((MIN_SURROGATE <= c && c <= MAX_SURROGATE) + && (inIx + 1 == inLimit || !isSurrogatePair(c, in.charAt(inIx + 1)))) { + // We are surrogates and we're not a surrogate pair. + throw new UnpairedSurrogateException(inIx, inLimit); + } + // Not enough space in the output buffer. + throw new ArrayIndexOutOfBoundsException("Failed writing " + c + " at index " + outIx); + } + } + + // All bytes have been encoded. + out.position((int) (outIx - address)); + } + + /** + * Counts (approximately) the number of consecutive ASCII characters starting from the given + * position, using the most efficient method available to the platform. + * + * @param bytes the array containing the character sequence + * @param offset the offset position of the index (same as index + arrayBaseOffset) + * @param maxChars the maximum number of characters to count + * @return the number of ASCII characters found. The stopping position will be at or + * before the first non-ASCII byte. + */ + private static int unsafeEstimateConsecutiveAscii( + byte[] bytes, long offset, final int maxChars) { + int remaining = maxChars; + if (remaining < UNSAFE_COUNT_ASCII_THRESHOLD) { + // Don't bother with small strings. + return 0; + } + + // Read bytes until 8-byte aligned so that we can read longs in the loop below. + // Byte arrays are already either 8 or 16-byte aligned, so we just need to make sure that + // the index (relative to the start of the array) is also 8-byte aligned. We do this by + // ANDing the index with 7 to determine the number of bytes that need to be read before + // we're 8-byte aligned. + final int unaligned = (int) offset & 7; + for (int j = unaligned; j > 0; j--) { + if (UNSAFE.getByte(bytes, offset++) < 0) { + return unaligned - j; + } + } + + // This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII). + // To speed things up further, we're reading longs instead of bytes so we use a mask to + // determine if any byte in the current long is non-ASCII. + remaining -= unaligned; + for (; remaining >= 8 && (UNSAFE.getLong(bytes, offset) & ASCII_MASK_LONG) == 0; + offset += 8, remaining -= 8) {} + return maxChars - remaining; + } + + /** + * Same as {@link Utf8#estimateConsecutiveAscii(ByteBuffer, int, int)} except that it uses the + * most efficient method available to the platform. + */ + private static int unsafeEstimateConsecutiveAscii(long address, final int maxChars) { + int remaining = maxChars; + if (remaining < UNSAFE_COUNT_ASCII_THRESHOLD) { + // Don't bother with small strings. + return 0; + } + + // Read bytes until 8-byte aligned so that we can read longs in the loop below. + // We do this by ANDing the address with 7 to determine the number of bytes that need to + // be read before we're 8-byte aligned. + final int unaligned = (int) address & 7; + for (int j = unaligned; j > 0; j--) { + if (UNSAFE.getByte(address++) < 0) { + return unaligned - j; + } + } + + // This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII). + // To speed things up further, we're reading longs instead of bytes so we use a mask to + // determine if any byte in the current long is non-ASCII. + remaining -= unaligned; + for (; remaining >= 8 && (UNSAFE.getLong(address) & ASCII_MASK_LONG) == 0; + address += 8, remaining -= 8) {} + return maxChars - remaining; + } + + private static int partialIsValidUtf8(final byte[] bytes, long offset, int remaining) { + // Skip past ASCII characters as quickly as possible. + final int skipped = unsafeEstimateConsecutiveAscii(bytes, offset, remaining); + remaining -= skipped; + offset += skipped; + + for (;;) { + // Optimize for interior runs of ASCII bytes. + // TODO(nathanmittler): Consider checking 8 bytes at a time after some threshold? + // Maybe after seeing a few in a row that are ASCII, go back to fast mode? + int byte1 = 0; + for (; remaining > 0 && (byte1 = UNSAFE.getByte(bytes, offset++)) >= 0; --remaining) { + } + if (remaining == 0) { + return COMPLETE; + } + remaining--; + + // If we're here byte1 is not ASCII. Only need to handle 2-4 byte forms. + if (byte1 < (byte) 0xE0) { + // Two-byte form (110xxxxx 10xxxxxx) + if (remaining == 0) { + // Incomplete sequence + return byte1; + } + remaining--; + + // Simultaneously checks for illegal trailing-byte in + // leading position and overlong 2-byte form. + if (byte1 < (byte) 0xC2 + || UNSAFE.getByte(bytes, offset++) > (byte) 0xBF) { + return MALFORMED; + } + } else if (byte1 < (byte) 0xF0) { + // Three-byte form (1110xxxx 10xxxxxx 10xxxxxx) + if (remaining < 2) { + // Incomplete sequence + return unsafeIncompleteStateFor(bytes, byte1, offset, remaining); + } + remaining -= 2; + + final int byte2; + if ((byte2 = UNSAFE.getByte(bytes, offset++)) > (byte) 0xBF + // overlong? 5 most significant bits must not all be zero + || (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) + // check for illegal surrogate codepoints + || (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) + // byte3 trailing-byte test + || UNSAFE.getByte(bytes, offset++) > (byte) 0xBF) { + return MALFORMED; + } + } else { + // Four-byte form (1110xxxx 10xxxxxx 10xxxxxx 10xxxxxx) + if (remaining < 3) { + // Incomplete sequence + return unsafeIncompleteStateFor(bytes, byte1, offset, remaining); + } + remaining -= 3; + + final int byte2; + if ((byte2 = UNSAFE.getByte(bytes, offset++)) > (byte) 0xBF + // Check that 1 <= plane <= 16. Tricky optimized form of: + // if (byte1 > (byte) 0xF4 || + // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || + // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) + || (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 + // byte3 trailing-byte test + || UNSAFE.getByte(bytes, offset++) > (byte) 0xBF + // byte4 trailing-byte test + || UNSAFE.getByte(bytes, offset++) > (byte) 0xBF) { + return MALFORMED; + } + } + } + } + + private static int partialIsValidUtf8(long address, int remaining) { + // Skip past ASCII characters as quickly as possible. + final int skipped = unsafeEstimateConsecutiveAscii(address, remaining); + address += skipped; + remaining -= skipped; + + for (;;) { + // Optimize for interior runs of ASCII bytes. + // TODO(nathanmittler): Consider checking 8 bytes at a time after some threshold? + // Maybe after seeing a few in a row that are ASCII, go back to fast mode? + int byte1 = 0; + for (; remaining > 0 && (byte1 = UNSAFE.getByte(address++)) >= 0; --remaining) { + } + if (remaining == 0) { + return COMPLETE; + } + remaining--; + + if (byte1 < (byte) 0xE0) { + // Two-byte form + + if (remaining == 0) { + // Incomplete sequence + return byte1; + } + remaining--; + + // Simultaneously checks for illegal trailing-byte in + // leading position and overlong 2-byte form. + if (byte1 < (byte) 0xC2 || UNSAFE.getByte(address++) > (byte) 0xBF) { + return MALFORMED; + } + } else if (byte1 < (byte) 0xF0) { + // Three-byte form + + if (remaining < 2) { + // Incomplete sequence + return unsafeIncompleteStateFor(address, byte1, remaining); + } + remaining -= 2; + + final byte byte2 = UNSAFE.getByte(address++); + if (byte2 > (byte) 0xBF + // overlong? 5 most significant bits must not all be zero + || (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) + // check for illegal surrogate codepoints + || (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) + // byte3 trailing-byte test + || UNSAFE.getByte(address++) > (byte) 0xBF) { + return MALFORMED; + } + } else { + // Four-byte form + + if (remaining < 3) { + // Incomplete sequence + return unsafeIncompleteStateFor(address, byte1, remaining); + } + remaining -= 3; + + final byte byte2 = UNSAFE.getByte(address++); + if (byte2 > (byte) 0xBF + // Check that 1 <= plane <= 16. Tricky optimized form of: + // if (byte1 > (byte) 0xF4 || + // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || + // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) + || (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 + // byte3 trailing-byte test + || UNSAFE.getByte(address++) > (byte) 0xBF + // byte4 trailing-byte test + || UNSAFE.getByte(address++) > (byte) 0xBF) { + return MALFORMED; + } + } + } + } + + private static int unsafeIncompleteStateFor(byte[] bytes, int byte1, long offset, + int remaining) { + switch (remaining) { + case 0: { + return incompleteStateFor(byte1); + } + case 1: { + return incompleteStateFor(byte1, UNSAFE.getByte(bytes, offset)); + } + case 2: { + return incompleteStateFor(byte1, UNSAFE.getByte(bytes, offset), + UNSAFE.getByte(bytes, offset + 1)); + } + default: { + throw new AssertionError(); + } + } + } + + private static int unsafeIncompleteStateFor(long address, final int byte1, int remaining) { + switch (remaining) { + case 0: { + return incompleteStateFor(byte1); + } + case 1: { + return incompleteStateFor(byte1, UNSAFE.getByte(address)); + } + case 2: { + return incompleteStateFor(byte1, UNSAFE.getByte(address), UNSAFE.getByte(address + 1)); + } + default: { + throw new AssertionError(); + } + } + } + + /** + * Gets the field with the given name within the class, or {@code null} if not found. If + * found, the field is made accessible. + */ + private static Field field(Class<?> clazz, String fieldName) { + Field field; + try { + field = clazz.getDeclaredField(fieldName); + field.setAccessible(true); + } catch (Throwable t) { + // Failed to access the fields. + field = null; + } + logger.log(Level.FINEST, "{0}.{1}: {2}", + new Object[] {clazz.getName(), fieldName, (field != null ? "available" : "unavailable")}); + return field; + } + + /** + * Returns the offset of the provided field, or {@code -1} if {@code sun.misc.Unsafe} is not + * available. + */ + private static long fieldOffset(Field field) { + return field == null || UNSAFE == null ? -1 : UNSAFE.objectFieldOffset(field); + } + + /** + * Get the base offset for byte arrays, or {@code -1} if {@code sun.misc.Unsafe} is not + * available. + */ + private static <T> int byteArrayBaseOffset() { + return UNSAFE == null ? -1 : UNSAFE.arrayBaseOffset(byte[].class); + } + + /** + * Gets the offset of the {@code address} field of the given direct {@link ByteBuffer}. + */ + private static long addressOffset(ByteBuffer buffer) { + return UNSAFE.getLong(buffer, BUFFER_ADDRESS_OFFSET); + } + + /** + * Gets the {@code sun.misc.Unsafe} instance, or {@code null} if not available on this + * platform. + */ + private static sun.misc.Unsafe getUnsafe() { + sun.misc.Unsafe unsafe = null; + try { + unsafe = AccessController.doPrivileged(new PrivilegedExceptionAction<sun.misc.Unsafe>() { + @Override + public sun.misc.Unsafe run() throws Exception { + Class<sun.misc.Unsafe> k = sun.misc.Unsafe.class; + + // Check that this platform supports all of the required unsafe methods. + checkRequiredMethods(k); + + for (Field f : k.getDeclaredFields()) { + f.setAccessible(true); + Object x = f.get(null); + if (k.isInstance(x)) { + return k.cast(x); + } + } + // The sun.misc.Unsafe field does not exist. + return null; + } + }); + } catch (Throwable e) { + // Catching Throwable here due to the fact that Google AppEngine raises NoClassDefFoundError + // for Unsafe. + } + + logger.log(Level.FINEST, "sun.misc.Unsafe: {}", + unsafe != null ? "available" : "unavailable"); + return unsafe; + } + + /** + * Verifies that all required methods of {@code sun.misc.Unsafe} are available on this platform. + */ + private static void checkRequiredMethods(Class<sun.misc.Unsafe> clazz) + throws NoSuchMethodException, SecurityException { + // Needed for Unsafe byte[] access + clazz.getMethod("arrayBaseOffset", Class.class); + clazz.getMethod("getByte", Object.class, long.class); + clazz.getMethod("putByte", Object.class, long.class, byte.class); + clazz.getMethod("getLong", Object.class, long.class); + + // Needed for Unsafe Direct ByteBuffer access + clazz.getMethod("objectFieldOffset", Field.class); + clazz.getMethod("getByte", long.class); + clazz.getMethod("getLong", Object.class, long.class); + clazz.getMethod("putByte", long.class, byte.class); + clazz.getMethod("getLong", long.class); + } + } + + private Utf8() {} } diff --git a/java/core/src/test/java/com/google/protobuf/AbstractMessageTest.java b/java/core/src/test/java/com/google/protobuf/AbstractMessageTest.java index d964ef59..1e57b647 100644 --- a/java/core/src/test/java/com/google/protobuf/AbstractMessageTest.java +++ b/java/core/src/test/java/com/google/protobuf/AbstractMessageTest.java @@ -40,7 +40,6 @@ import protobuf_unittest.UnittestProto.TestPackedTypes; import protobuf_unittest.UnittestProto.TestRequired; import protobuf_unittest.UnittestProto.TestRequiredForeign; import protobuf_unittest.UnittestProto.TestUnpackedTypes; - import junit.framework.TestCase; import java.util.Map; diff --git a/java/core/src/test/java/com/google/protobuf/AnyTest.java b/java/core/src/test/java/com/google/protobuf/AnyTest.java index e169f69d..cf91ed91 100644 --- a/java/core/src/test/java/com/google/protobuf/AnyTest.java +++ b/java/core/src/test/java/com/google/protobuf/AnyTest.java @@ -75,6 +75,51 @@ public class AnyTest extends TestCase { } } + public void testCustomTypeUrls() throws Exception { + TestAllTypes.Builder builder = TestAllTypes.newBuilder(); + TestUtil.setAllFields(builder); + TestAllTypes message = builder.build(); + + TestAny container = TestAny.newBuilder() + .setValue(Any.pack(message, "xxx.com")).build(); + + assertEquals( + "xxx.com/" + TestAllTypes.getDescriptor().getFullName(), + container.getValue().getTypeUrl()); + + assertTrue(container.getValue().is(TestAllTypes.class)); + assertFalse(container.getValue().is(TestAny.class)); + + TestAllTypes result = container.getValue().unpack(TestAllTypes.class); + TestUtil.assertAllFieldsSet(result); + + container = TestAny.newBuilder() + .setValue(Any.pack(message, "yyy.com/")).build(); + + assertEquals( + "yyy.com/" + TestAllTypes.getDescriptor().getFullName(), + container.getValue().getTypeUrl()); + + assertTrue(container.getValue().is(TestAllTypes.class)); + assertFalse(container.getValue().is(TestAny.class)); + + result = container.getValue().unpack(TestAllTypes.class); + TestUtil.assertAllFieldsSet(result); + + container = TestAny.newBuilder() + .setValue(Any.pack(message, "")).build(); + + assertEquals( + "/" + TestAllTypes.getDescriptor().getFullName(), + container.getValue().getTypeUrl()); + + assertTrue(container.getValue().is(TestAllTypes.class)); + assertFalse(container.getValue().is(TestAny.class)); + + result = container.getValue().unpack(TestAllTypes.class); + TestUtil.assertAllFieldsSet(result); + } + public void testCachedUnpackResult() throws Exception { TestAllTypes.Builder builder = TestAllTypes.newBuilder(); TestUtil.setAllFields(builder); diff --git a/java/core/src/test/java/com/google/protobuf/BoundedByteStringTest.java b/java/core/src/test/java/com/google/protobuf/BoundedByteStringTest.java index 7a8a0a5e..db10ee74 100644 --- a/java/core/src/test/java/com/google/protobuf/BoundedByteStringTest.java +++ b/java/core/src/test/java/com/google/protobuf/BoundedByteStringTest.java @@ -37,7 +37,6 @@ import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.UnsupportedEncodingException; - /** * This class tests {@link BoundedByteString}, which extends {@link LiteralByteString}, * by inheriting the tests from {@link LiteralByteStringTest}. The only method which diff --git a/java/core/src/test/java/com/google/protobuf/ByteBufferWriterTest.java b/java/core/src/test/java/com/google/protobuf/ByteBufferWriterTest.java new file mode 100644 index 00000000..cbe742e5 --- /dev/null +++ b/java/core/src/test/java/com/google/protobuf/ByteBufferWriterTest.java @@ -0,0 +1,81 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import junit.framework.TestCase; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Random; + +/** + * Tests for {@link ByteBufferWriter}. + */ +public class ByteBufferWriterTest extends TestCase { + + public void testHeapBuffer() throws IOException { + // Test a small and large buffer. + testWrite(ByteBuffer.allocate(100)); + testWrite(ByteBuffer.allocate(1024 * 100)); + } + + public void testDirectBuffer() throws IOException { + // Test a small and large buffer. + testWrite(ByteBuffer.allocateDirect(100)); + testWrite(ByteBuffer.allocateDirect(1024 * 100)); + } + + private void testWrite(ByteBuffer buffer) throws IOException { + fillRandom(buffer); + ByteArrayOutputStream os = new ByteArrayOutputStream(buffer.remaining()); + ByteBufferWriter.write(buffer, os); + assertEquals(0, buffer.position()); + assertTrue(Arrays.equals(toArray(buffer), os.toByteArray())); + } + + private void fillRandom(ByteBuffer buf) { + byte[] bytes = new byte[buf.remaining()]; + new Random().nextBytes(bytes); + buf.put(bytes); + buf.flip(); + return; + } + + private byte[] toArray(ByteBuffer buf) { + int originalPosition = buf.position(); + byte[] bytes = new byte[buf.remaining()]; + buf.get(bytes); + buf.position(originalPosition); + return bytes; + } +} diff --git a/java/core/src/test/java/com/google/protobuf/ByteStringTest.java b/java/core/src/test/java/com/google/protobuf/ByteStringTest.java index 5267c160..ad9f266e 100644 --- a/java/core/src/test/java/com/google/protobuf/ByteStringTest.java +++ b/java/core/src/test/java/com/google/protobuf/ByteStringTest.java @@ -39,6 +39,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.util.ArrayList; @@ -757,4 +758,17 @@ public class ByteStringTest extends TestCase { assertEquals((byte) 2, result[dataSize - dataSize / 2]); assertEquals((byte) 2, result[dataSize - 1]); } + + /** + * Tests ByteString uses Arrays based byte copier when running under Hotstop VM. + */ + public void testByteArrayCopier() throws Exception { + Field field = ByteString.class.getDeclaredField("byteArrayCopier"); + field.setAccessible(true); + Object byteArrayCopier = field.get(null); + assertNotNull(byteArrayCopier); + assertTrue( + byteArrayCopier.toString(), + byteArrayCopier.getClass().getSimpleName().endsWith("ArraysByteArrayCopier")); + } } diff --git a/java/core/src/test/java/com/google/protobuf/CheckUtf8Test.java b/java/core/src/test/java/com/google/protobuf/CheckUtf8Test.java index 3d6381c9..cc65d19a 100644 --- a/java/core/src/test/java/com/google/protobuf/CheckUtf8Test.java +++ b/java/core/src/test/java/com/google/protobuf/CheckUtf8Test.java @@ -34,6 +34,7 @@ import proto2_test_check_utf8.TestCheckUtf8.BytesWrapper; import proto2_test_check_utf8.TestCheckUtf8.StringWrapper; import proto2_test_check_utf8_size.TestCheckUtf8Size.BytesWrapperSize; import proto2_test_check_utf8_size.TestCheckUtf8Size.StringWrapperSize; + import junit.framework.TestCase; /** diff --git a/java/core/src/test/java/com/google/protobuf/CodedInputStreamTest.java b/java/core/src/test/java/com/google/protobuf/CodedInputStreamTest.java index 18d8142c..a1d6f1be 100644 --- a/java/core/src/test/java/com/google/protobuf/CodedInputStreamTest.java +++ b/java/core/src/test/java/com/google/protobuf/CodedInputStreamTest.java @@ -547,6 +547,56 @@ public class CodedInputStreamTest extends TestCase { } } + public void testReadString() throws Exception { + String lorem = "Lorem ipsum dolor sit amet "; + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < 4096; i += lorem.length()) { + builder.append(lorem); + } + lorem = builder.toString().substring(0, 4096); + byte[] bytes = lorem.getBytes("UTF-8"); + ByteString.Output rawOutput = ByteString.newOutput(); + CodedOutputStream output = CodedOutputStream.newInstance(rawOutput, bytes.length); + + int tag = WireFormat.makeTag(1, WireFormat.WIRETYPE_LENGTH_DELIMITED); + output.writeRawVarint32(tag); + output.writeRawVarint32(bytes.length); + output.writeRawBytes(bytes); + output.flush(); + + CodedInputStream input = + CodedInputStream.newInstance( + new ByteArrayInputStream(rawOutput.toByteString().toByteArray())); + assertEquals(tag, input.readTag()); + String text = input.readString(); + assertEquals(lorem, text); + } + + public void testReadStringRequireUtf8() throws Exception { + String lorem = "Lorem ipsum dolor sit amet "; + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < 4096; i += lorem.length()) { + builder.append(lorem); + } + lorem = builder.toString().substring(0, 4096); + byte[] bytes = lorem.getBytes("UTF-8"); + ByteString.Output rawOutput = ByteString.newOutput(); + CodedOutputStream output = CodedOutputStream.newInstance(rawOutput, bytes.length); + + int tag = WireFormat.makeTag(1, WireFormat.WIRETYPE_LENGTH_DELIMITED); + output.writeRawVarint32(tag); + output.writeRawVarint32(bytes.length); + output.writeRawBytes(bytes); + output.flush(); + + CodedInputStream input = + CodedInputStream.newInstance( + new ByteArrayInputStream(rawOutput.toByteString().toByteArray())); + assertEquals(tag, input.readTag()); + String text = input.readStringRequireUtf8(); + assertEquals(lorem, text); + } + /** * Tests that if we readString invalid UTF-8 bytes, no exception * is thrown. Instead, the invalid bytes are replaced with the Unicode diff --git a/java/core/src/test/java/com/google/protobuf/DeprecatedFieldTest.java b/java/core/src/test/java/com/google/protobuf/DeprecatedFieldTest.java index e7905f79..ce85ae2e 100644 --- a/java/core/src/test/java/com/google/protobuf/DeprecatedFieldTest.java +++ b/java/core/src/test/java/com/google/protobuf/DeprecatedFieldTest.java @@ -36,6 +36,7 @@ import junit.framework.TestCase; import java.lang.reflect.AnnotatedElement; import java.lang.reflect.Method; + /** * Test field deprecation * diff --git a/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java b/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java index 82ff34af..ef89b389 100644 --- a/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java +++ b/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java @@ -59,7 +59,6 @@ import protobuf_unittest.UnittestProto.TestMultipleExtensionRanges; import protobuf_unittest.UnittestProto.TestRequired; import protobuf_unittest.UnittestProto.TestReservedFields; import protobuf_unittest.UnittestProto.TestService; - import junit.framework.TestCase; import java.util.Arrays; @@ -573,6 +572,42 @@ public class DescriptorsTest extends TestCase { } } + public void testUnknownFieldsDenied() throws Exception { + FileDescriptorProto fooProto = FileDescriptorProto.newBuilder() + .setName("foo.proto") + .addMessageType(DescriptorProto.newBuilder() + .setName("Foo") + .addField(FieldDescriptorProto.newBuilder() + .setLabel(FieldDescriptorProto.Label.LABEL_OPTIONAL) + .setTypeName("Bar") + .setName("bar") + .setNumber(1))) + .build(); + + try { + Descriptors.FileDescriptor.buildFrom(fooProto, new FileDescriptor[0]); + fail("DescriptorValidationException expected"); + } catch (DescriptorValidationException e) { + assertTrue(e.getMessage().indexOf("Bar") != -1); + assertTrue(e.getMessage().indexOf("is not defined") != -1); + } + } + + public void testUnknownFieldsAllowed() throws Exception { + FileDescriptorProto fooProto = FileDescriptorProto.newBuilder() + .setName("foo.proto") + .addDependency("bar.proto") + .addMessageType(DescriptorProto.newBuilder() + .setName("Foo") + .addField(FieldDescriptorProto.newBuilder() + .setLabel(FieldDescriptorProto.Label.LABEL_OPTIONAL) + .setTypeName("Bar") + .setName("bar") + .setNumber(1))) + .build(); + Descriptors.FileDescriptor.buildFrom(fooProto, new FileDescriptor[0], true); + } + public void testHiddenDependency() throws Exception { FileDescriptorProto barProto = FileDescriptorProto.newBuilder() .setName("bar.proto") diff --git a/java/core/src/test/java/com/google/protobuf/DynamicMessageTest.java b/java/core/src/test/java/com/google/protobuf/DynamicMessageTest.java index 55144e7c..8f3a74c1 100644 --- a/java/core/src/test/java/com/google/protobuf/DynamicMessageTest.java +++ b/java/core/src/test/java/com/google/protobuf/DynamicMessageTest.java @@ -33,13 +33,13 @@ package com.google.protobuf; import com.google.protobuf.Descriptors.EnumDescriptor; import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.Descriptors.OneofDescriptor; - import protobuf_unittest.UnittestProto.TestAllExtensions; import protobuf_unittest.UnittestProto.TestAllTypes; import protobuf_unittest.UnittestProto.TestEmptyMessage; import protobuf_unittest.UnittestProto.TestPackedTypes; import junit.framework.TestCase; + import java.util.Arrays; /** diff --git a/java/core/src/test/java/com/google/protobuf/EnumTest.java b/java/core/src/test/java/com/google/protobuf/EnumTest.java new file mode 100644 index 00000000..14c7406b --- /dev/null +++ b/java/core/src/test/java/com/google/protobuf/EnumTest.java @@ -0,0 +1,76 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import com.google.protobuf.UnittestLite.ForeignEnumLite; +import com.google.protobuf.UnittestLite.TestAllTypesLite; +import protobuf_unittest.UnittestProto.ForeignEnum; +import protobuf_unittest.UnittestProto.TestAllTypes; + +import junit.framework.TestCase; + +public class EnumTest extends TestCase { + + public void testForNumber() { + ForeignEnum e = ForeignEnum.forNumber(ForeignEnum.FOREIGN_BAR.getNumber()); + assertEquals(ForeignEnum.FOREIGN_BAR, e); + + e = ForeignEnum.forNumber(1000); + assertEquals(null, e); + } + + public void testForNumber_oneof() { + TestAllTypes.OneofFieldCase e = TestAllTypes.OneofFieldCase.forNumber( + TestAllTypes.OneofFieldCase.ONEOF_NESTED_MESSAGE.getNumber()); + assertEquals(TestAllTypes.OneofFieldCase.ONEOF_NESTED_MESSAGE, e); + + e = TestAllTypes.OneofFieldCase.forNumber(1000); + assertEquals(null, e); + } + + public void testForNumberLite() { + ForeignEnumLite e = ForeignEnumLite.forNumber(ForeignEnumLite.FOREIGN_LITE_BAR.getNumber()); + assertEquals(ForeignEnumLite.FOREIGN_LITE_BAR, e); + + e = ForeignEnumLite.forNumber(1000); + assertEquals(null, e); + } + + public void testForNumberLite_oneof() { + TestAllTypesLite.OneofFieldCase e = TestAllTypesLite.OneofFieldCase.forNumber( + TestAllTypesLite.OneofFieldCase.ONEOF_NESTED_MESSAGE.getNumber()); + assertEquals(TestAllTypesLite.OneofFieldCase.ONEOF_NESTED_MESSAGE, e); + + e = TestAllTypesLite.OneofFieldCase.forNumber(1000); + assertEquals(null, e); + } +} + diff --git a/java/core/src/test/java/com/google/protobuf/FieldPresenceTest.java b/java/core/src/test/java/com/google/protobuf/FieldPresenceTest.java index eaeec0b8..304cec4f 100644 --- a/java/core/src/test/java/com/google/protobuf/FieldPresenceTest.java +++ b/java/core/src/test/java/com/google/protobuf/FieldPresenceTest.java @@ -44,9 +44,9 @@ import junit.framework.TestCase; * non-message fields. */ public class FieldPresenceTest extends TestCase { - private static boolean hasMethod(Class clazz, String name) { + private static boolean hasMethod(Class<?> clazz, String name) { try { - if (clazz.getMethod(name, new Class[]{}) != null) { + if (clazz.getMethod(name) != null) { return true; } else { return false; @@ -56,90 +56,90 @@ public class FieldPresenceTest extends TestCase { } } - private static boolean isHasMethodRemoved( - Class classWithFieldPresence, - Class classWithoutFieldPresence, + private static void assertHasMethodRemoved( + Class<?> classWithFieldPresence, + Class<?> classWithoutFieldPresence, String camelName) { - return hasMethod(classWithFieldPresence, "get" + camelName) - && hasMethod(classWithFieldPresence, "has" + camelName) - && hasMethod(classWithoutFieldPresence, "get" + camelName) - && !hasMethod(classWithoutFieldPresence, "has" + camelName); + assertTrue(hasMethod(classWithFieldPresence, "get" + camelName)); + assertTrue(hasMethod(classWithFieldPresence, "has" + camelName)); + assertTrue(hasMethod(classWithoutFieldPresence, "get" + camelName)); + assertFalse(hasMethod(classWithoutFieldPresence, "has" + camelName)); } public void testHasMethod() { // Optional non-message fields don't have a hasFoo() method generated. - assertTrue(isHasMethodRemoved( + assertHasMethodRemoved( UnittestProto.TestAllTypes.class, TestAllTypes.class, - "OptionalInt32")); - assertTrue(isHasMethodRemoved( + "OptionalInt32"); + assertHasMethodRemoved( UnittestProto.TestAllTypes.class, TestAllTypes.class, - "OptionalString")); - assertTrue(isHasMethodRemoved( + "OptionalString"); + assertHasMethodRemoved( UnittestProto.TestAllTypes.class, TestAllTypes.class, - "OptionalBytes")); - assertTrue(isHasMethodRemoved( + "OptionalBytes"); + assertHasMethodRemoved( UnittestProto.TestAllTypes.class, TestAllTypes.class, - "OptionalNestedEnum")); + "OptionalNestedEnum"); - assertTrue(isHasMethodRemoved( + assertHasMethodRemoved( UnittestProto.TestAllTypes.Builder.class, TestAllTypes.Builder.class, - "OptionalInt32")); - assertTrue(isHasMethodRemoved( + "OptionalInt32"); + assertHasMethodRemoved( UnittestProto.TestAllTypes.Builder.class, TestAllTypes.Builder.class, - "OptionalString")); - assertTrue(isHasMethodRemoved( + "OptionalString"); + assertHasMethodRemoved( UnittestProto.TestAllTypes.Builder.class, TestAllTypes.Builder.class, - "OptionalBytes")); - assertTrue(isHasMethodRemoved( + "OptionalBytes"); + assertHasMethodRemoved( UnittestProto.TestAllTypes.Builder.class, TestAllTypes.Builder.class, - "OptionalNestedEnum")); + "OptionalNestedEnum"); // message fields still have the hasFoo() method generated. assertFalse(TestAllTypes.newBuilder().build().hasOptionalNestedMessage()); assertFalse(TestAllTypes.newBuilder().hasOptionalNestedMessage()); // oneof fields don't have hasFoo() methods (even for message types). - assertTrue(isHasMethodRemoved( + assertHasMethodRemoved( UnittestProto.TestAllTypes.class, TestAllTypes.class, - "OneofUint32")); - assertTrue(isHasMethodRemoved( + "OneofUint32"); + assertHasMethodRemoved( UnittestProto.TestAllTypes.class, TestAllTypes.class, - "OneofString")); - assertTrue(isHasMethodRemoved( + "OneofString"); + assertHasMethodRemoved( UnittestProto.TestAllTypes.class, TestAllTypes.class, - "OneofBytes")); - assertTrue(isHasMethodRemoved( + "OneofBytes"); + assertHasMethodRemoved( UnittestProto.TestAllTypes.class, TestAllTypes.class, - "OneofNestedMessage")); + "OneofNestedMessage"); - assertTrue(isHasMethodRemoved( + assertHasMethodRemoved( UnittestProto.TestAllTypes.Builder.class, TestAllTypes.Builder.class, - "OneofUint32")); - assertTrue(isHasMethodRemoved( + "OneofUint32"); + assertHasMethodRemoved( UnittestProto.TestAllTypes.Builder.class, TestAllTypes.Builder.class, - "OneofString")); - assertTrue(isHasMethodRemoved( + "OneofString"); + assertHasMethodRemoved( UnittestProto.TestAllTypes.Builder.class, TestAllTypes.Builder.class, - "OneofBytes")); - assertTrue(isHasMethodRemoved( + "OneofBytes"); + assertHasMethodRemoved( UnittestProto.TestAllTypes.Builder.class, TestAllTypes.Builder.class, - "OneofNestedMessage")); + "OneofNestedMessage"); } public void testOneofEquals() throws Exception { @@ -232,24 +232,24 @@ public class FieldPresenceTest extends TestCase { assertTrue(message.hasField(optionalNestedEnumField)); assertEquals(4, message.getAllFields().size()); } - + public void testMessageField() { TestAllTypes.Builder builder = TestAllTypes.newBuilder(); assertFalse(builder.hasOptionalNestedMessage()); assertFalse(builder.build().hasOptionalNestedMessage()); - + TestAllTypes.NestedMessage.Builder nestedBuilder = builder.getOptionalNestedMessageBuilder(); assertTrue(builder.hasOptionalNestedMessage()); assertTrue(builder.build().hasOptionalNestedMessage()); - + nestedBuilder.setValue(1); assertEquals(1, builder.build().getOptionalNestedMessage().getValue()); - + builder.clearOptionalNestedMessage(); assertFalse(builder.hasOptionalNestedMessage()); assertFalse(builder.build().hasOptionalNestedMessage()); - + // Unlike non-message fields, if we set a message field to its default value (i.e., // default instance), the field should be seen as present. builder.setOptionalNestedMessage(TestAllTypes.NestedMessage.getDefaultInstance()); @@ -340,7 +340,7 @@ public class FieldPresenceTest extends TestCase { assertTrue(builder.buildPartial().isInitialized()); } - + // Test that unknown fields are dropped. public void testUnknownFields() throws Exception { TestAllTypes.Builder builder = TestAllTypes.newBuilder(); @@ -348,7 +348,7 @@ public class FieldPresenceTest extends TestCase { builder.addRepeatedInt32(5678); TestAllTypes message = builder.build(); ByteString data = message.toByteString(); - + TestOptionalFieldsOnly optionalOnlyMessage = TestOptionalFieldsOnly.parseFrom(data); // UnknownFieldSet should be empty. @@ -360,7 +360,7 @@ public class FieldPresenceTest extends TestCase { // The repeated field is discarded because it's unknown to the optional-only // message. assertEquals(0, message.getRepeatedInt32Count()); - + DynamicMessage dynamicOptionalOnlyMessage = DynamicMessage.getDefaultInstance( TestOptionalFieldsOnly.getDescriptor()) diff --git a/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java b/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java index 70812b95..8cd1f38d 100644 --- a/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java +++ b/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java @@ -620,6 +620,21 @@ public class GeneratedMessageTest extends TestCase { TestUtil.assertExtensionsClear(TestAllExtensions.newBuilder().build()); } + public void testUnsetRepeatedExtensionGetField() { + TestAllExtensions message = TestAllExtensions.getDefaultInstance(); + Object value; + + value = message.getField(UnittestProto.repeatedStringExtension.getDescriptor()); + assertTrue(value instanceof List); + assertTrue(((List<?>) value).isEmpty()); + assertIsUnmodifiable((List<?>) value); + + value = message.getField(UnittestProto.repeatedNestedMessageExtension.getDescriptor()); + assertTrue(value instanceof List); + assertTrue(((List<?>) value).isEmpty()); + assertIsUnmodifiable((List<?>) value); + } + public void testExtensionReflectionGetters() throws Exception { TestAllExtensions.Builder builder = TestAllExtensions.newBuilder(); TestUtil.setAllExtensions(builder); diff --git a/java/core/src/test/java/com/google/protobuf/IsValidUtf8Test.java b/java/core/src/test/java/com/google/protobuf/IsValidUtf8Test.java index 8751baae..756049b4 100644 --- a/java/core/src/test/java/com/google/protobuf/IsValidUtf8Test.java +++ b/java/core/src/test/java/com/google/protobuf/IsValidUtf8Test.java @@ -30,12 +30,18 @@ package com.google.protobuf; +import static com.google.protobuf.IsValidUtf8TestUtil.DIRECT_NIO_FACTORY; +import static com.google.protobuf.IsValidUtf8TestUtil.EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT; +import static com.google.protobuf.IsValidUtf8TestUtil.EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT; +import static com.google.protobuf.IsValidUtf8TestUtil.HEAP_NIO_FACTORY; +import static com.google.protobuf.IsValidUtf8TestUtil.LITERAL_FACTORY; +import static com.google.protobuf.IsValidUtf8TestUtil.testBytes; + +import com.google.protobuf.IsValidUtf8TestUtil.ByteStringFactory; import com.google.protobuf.IsValidUtf8TestUtil.Shard; import junit.framework.TestCase; -import java.io.UnsupportedEncodingException; - /** * Tests cases for {@link ByteString#isValidUtf8()}. This includes three * brute force tests that actually test every permutation of one byte, two byte, @@ -51,31 +57,33 @@ import java.io.UnsupportedEncodingException; * @author martinrb@google.com (Martin Buchholz) */ public class IsValidUtf8Test extends TestCase { - /** * Tests that round tripping of all two byte permutations work. */ - public void testIsValidUtf8_1Byte() throws UnsupportedEncodingException { - IsValidUtf8TestUtil.testBytes(1, - IsValidUtf8TestUtil.EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT); + public void testIsValidUtf8_1Byte() { + testBytes(LITERAL_FACTORY, 1, EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT); + testBytes(HEAP_NIO_FACTORY, 1, EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT); + testBytes(DIRECT_NIO_FACTORY, 1, EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT); } /** * Tests that round tripping of all two byte permutations work. */ - public void testIsValidUtf8_2Bytes() throws UnsupportedEncodingException { - IsValidUtf8TestUtil.testBytes(2, - IsValidUtf8TestUtil.EXPECTED_TWO_BYTE_ROUNDTRIPPABLE_COUNT); + public void testIsValidUtf8_2Bytes() { + testBytes(LITERAL_FACTORY, 2, IsValidUtf8TestUtil.EXPECTED_TWO_BYTE_ROUNDTRIPPABLE_COUNT); + testBytes(HEAP_NIO_FACTORY, 2, IsValidUtf8TestUtil.EXPECTED_TWO_BYTE_ROUNDTRIPPABLE_COUNT); + testBytes(DIRECT_NIO_FACTORY, 2, IsValidUtf8TestUtil.EXPECTED_TWO_BYTE_ROUNDTRIPPABLE_COUNT); } /** * Tests that round tripping of all three byte permutations work. */ - public void testIsValidUtf8_3Bytes() throws UnsupportedEncodingException { + public void testIsValidUtf8_3Bytes() { // Travis' OOM killer doesn't like this test if (System.getenv("TRAVIS") == null) { - IsValidUtf8TestUtil.testBytes(3, - IsValidUtf8TestUtil.EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT); + testBytes(LITERAL_FACTORY, 3, EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT); + testBytes(HEAP_NIO_FACTORY, 3, EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT); + testBytes(DIRECT_NIO_FACTORY, 3, EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT); } } @@ -85,8 +93,7 @@ public class IsValidUtf8Test extends TestCase { * {@link IsValidUtf8FourByteTest} is used for full coverage. This method * tests specific four-byte cases. */ - public void testIsValidUtf8_4BytesSamples() - throws UnsupportedEncodingException { + public void testIsValidUtf8_4BytesSamples() { // Valid 4 byte. assertValidUtf8(0xF0, 0xA4, 0xAD, 0xA2); @@ -119,9 +126,7 @@ public class IsValidUtf8Test extends TestCase { assertTrue(asBytes("\u024B62\u024B62").isValidUtf8()); // Mixed string - assertTrue( - asBytes("a\u020ac\u00a2b\\u024B62u020acc\u00a2de\u024B62") - .isValidUtf8()); + assertTrue(asBytes("a\u020ac\u00a2b\\u024B62u020acc\u00a2de\u024B62").isValidUtf8()); // Not a valid string assertInvalidUtf8(-1, 0, -1, 0); @@ -135,36 +140,35 @@ public class IsValidUtf8Test extends TestCase { return realBytes; } - private ByteString toByteString(int... bytes) { - return ByteString.copyFrom(toByteArray(bytes)); - } - - private void assertValidUtf8(int[] bytes, boolean not) { + private void assertValidUtf8(ByteStringFactory factory, int[] bytes, boolean not) { byte[] realBytes = toByteArray(bytes); assertTrue(not ^ Utf8.isValidUtf8(realBytes)); assertTrue(not ^ Utf8.isValidUtf8(realBytes, 0, bytes.length)); - ByteString lit = ByteString.copyFrom(realBytes); - ByteString sub = lit.substring(0, bytes.length); - assertTrue(not ^ lit.isValidUtf8()); + ByteString leaf = factory.newByteString(realBytes); + ByteString sub = leaf.substring(0, bytes.length); + assertTrue(not ^ leaf.isValidUtf8()); assertTrue(not ^ sub.isValidUtf8()); ByteString[] ropes = { - RopeByteString.newInstanceForTest(ByteString.EMPTY, lit), - RopeByteString.newInstanceForTest(ByteString.EMPTY, sub), - RopeByteString.newInstanceForTest(lit, ByteString.EMPTY), - RopeByteString.newInstanceForTest(sub, ByteString.EMPTY), - RopeByteString.newInstanceForTest(sub, lit) - }; + RopeByteString.newInstanceForTest(ByteString.EMPTY, leaf), + RopeByteString.newInstanceForTest(ByteString.EMPTY, sub), + RopeByteString.newInstanceForTest(leaf, ByteString.EMPTY), + RopeByteString.newInstanceForTest(sub, ByteString.EMPTY), + RopeByteString.newInstanceForTest(sub, leaf)}; for (ByteString rope : ropes) { assertTrue(not ^ rope.isValidUtf8()); } } private void assertValidUtf8(int... bytes) { - assertValidUtf8(bytes, false); + assertValidUtf8(LITERAL_FACTORY, bytes, false); + assertValidUtf8(HEAP_NIO_FACTORY, bytes, false); + assertValidUtf8(DIRECT_NIO_FACTORY, bytes, false); } private void assertInvalidUtf8(int... bytes) { - assertValidUtf8(bytes, true); + assertValidUtf8(LITERAL_FACTORY, bytes, true); + assertValidUtf8(HEAP_NIO_FACTORY, bytes, true); + assertValidUtf8(DIRECT_NIO_FACTORY, bytes, true); } private static ByteString asBytes(String s) { @@ -177,7 +181,6 @@ public class IsValidUtf8Test extends TestCase { for (Shard shard : IsValidUtf8TestUtil.FOUR_BYTE_SHARDS) { actual += shard.expected; } - assertEquals(IsValidUtf8TestUtil.EXPECTED_FOUR_BYTE_ROUNDTRIPPABLE_COUNT, - actual); + assertEquals(IsValidUtf8TestUtil.EXPECTED_FOUR_BYTE_ROUNDTRIPPABLE_COUNT, actual); } } diff --git a/java/core/src/test/java/com/google/protobuf/IsValidUtf8TestUtil.java b/java/core/src/test/java/com/google/protobuf/IsValidUtf8TestUtil.java index 321669f3..16a808bf 100644 --- a/java/core/src/test/java/com/google/protobuf/IsValidUtf8TestUtil.java +++ b/java/core/src/test/java/com/google/protobuf/IsValidUtf8TestUtil.java @@ -30,9 +30,13 @@ package com.google.protobuf; -import static junit.framework.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; -import java.io.UnsupportedEncodingException; +import java.lang.ref.SoftReference; import java.nio.ByteBuffer; import java.nio.CharBuffer; import java.nio.charset.CharsetDecoder; @@ -52,64 +56,105 @@ import java.util.logging.Logger; * @author jonp@google.com (Jon Perlow) * @author martinrb@google.com (Martin Buchholz) */ -class IsValidUtf8TestUtil { - private static Logger logger = Logger.getLogger( - IsValidUtf8TestUtil.class.getName()); +final class IsValidUtf8TestUtil { + private static Logger logger = Logger.getLogger(IsValidUtf8TestUtil.class.getName()); + + private IsValidUtf8TestUtil() {} + + static interface ByteStringFactory { + ByteString newByteString(byte[] bytes); + } + + static final ByteStringFactory LITERAL_FACTORY = new ByteStringFactory() { + @Override + public ByteString newByteString(byte[] bytes) { + return ByteString.wrap(bytes); + } + }; + + static final ByteStringFactory HEAP_NIO_FACTORY = new ByteStringFactory() { + @Override + public ByteString newByteString(byte[] bytes) { + return new NioByteString(ByteBuffer.wrap(bytes)); + } + }; + + private static ThreadLocal<SoftReference<ByteBuffer>> directBuffer = + new ThreadLocal<SoftReference<ByteBuffer>>(); + + /** + * Factory for direct {@link ByteBuffer} instances. To reduce direct memory usage, this + * uses a thread local direct buffer. This means that each call will overwrite the buffer's + * contents from the previous call, so the calling code must be careful not to continue using + * a buffer returned from a previous invocation. + */ + static final ByteStringFactory DIRECT_NIO_FACTORY = new ByteStringFactory() { + @Override + public ByteString newByteString(byte[] bytes) { + SoftReference<ByteBuffer> ref = directBuffer.get(); + ByteBuffer buffer = ref == null ? null : ref.get(); + if (buffer == null || buffer.capacity() < bytes.length) { + buffer = ByteBuffer.allocateDirect(bytes.length); + directBuffer.set(new SoftReference<ByteBuffer>(buffer)); + } + buffer.clear(); + buffer.put(bytes); + buffer.flip(); + return new NioByteString(buffer); + } + }; // 128 - [chars 0x0000 to 0x007f] - static long ONE_BYTE_ROUNDTRIPPABLE_CHARACTERS = 0x007f - 0x0000 + 1; + static final long ONE_BYTE_ROUNDTRIPPABLE_CHARACTERS = 0x007f - 0x0000 + 1; // 128 - static long EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT = - ONE_BYTE_ROUNDTRIPPABLE_CHARACTERS; + static final long EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT = ONE_BYTE_ROUNDTRIPPABLE_CHARACTERS; // 1920 [chars 0x0080 to 0x07FF] - static long TWO_BYTE_ROUNDTRIPPABLE_CHARACTERS = 0x07FF - 0x0080 + 1; + static final long TWO_BYTE_ROUNDTRIPPABLE_CHARACTERS = 0x07FF - 0x0080 + 1; // 18,304 - static long EXPECTED_TWO_BYTE_ROUNDTRIPPABLE_COUNT = + static final long EXPECTED_TWO_BYTE_ROUNDTRIPPABLE_COUNT = // Both bytes are one byte characters (long) Math.pow(EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT, 2) + // The possible number of two byte characters TWO_BYTE_ROUNDTRIPPABLE_CHARACTERS; // 2048 - static long THREE_BYTE_SURROGATES = 2 * 1024; + static final long THREE_BYTE_SURROGATES = 2 * 1024; // 61,440 [chars 0x0800 to 0xFFFF, minus surrogates] - static long THREE_BYTE_ROUNDTRIPPABLE_CHARACTERS = + static final long THREE_BYTE_ROUNDTRIPPABLE_CHARACTERS = 0xFFFF - 0x0800 + 1 - THREE_BYTE_SURROGATES; // 2,650,112 - static long EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT = + static final long EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT = // All one byte characters (long) Math.pow(EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT, 3) + // One two byte character and a one byte character - 2 * TWO_BYTE_ROUNDTRIPPABLE_CHARACTERS * - ONE_BYTE_ROUNDTRIPPABLE_CHARACTERS + - // Three byte characters + 2 * TWO_BYTE_ROUNDTRIPPABLE_CHARACTERS * ONE_BYTE_ROUNDTRIPPABLE_CHARACTERS + + // Three byte characters THREE_BYTE_ROUNDTRIPPABLE_CHARACTERS; // 1,048,576 [chars 0x10000L to 0x10FFFF] - static long FOUR_BYTE_ROUNDTRIPPABLE_CHARACTERS = 0x10FFFF - 0x10000L + 1; + static final long FOUR_BYTE_ROUNDTRIPPABLE_CHARACTERS = 0x10FFFF - 0x10000L + 1; // 289,571,839 - static long EXPECTED_FOUR_BYTE_ROUNDTRIPPABLE_COUNT = + static final long EXPECTED_FOUR_BYTE_ROUNDTRIPPABLE_COUNT = // All one byte characters (long) Math.pow(EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT, 4) + // One and three byte characters - 2 * THREE_BYTE_ROUNDTRIPPABLE_CHARACTERS * - ONE_BYTE_ROUNDTRIPPABLE_CHARACTERS + + 2 * THREE_BYTE_ROUNDTRIPPABLE_CHARACTERS * ONE_BYTE_ROUNDTRIPPABLE_CHARACTERS + // Two two byte characters TWO_BYTE_ROUNDTRIPPABLE_CHARACTERS * TWO_BYTE_ROUNDTRIPPABLE_CHARACTERS + // Permutations of one and two byte characters - 3 * TWO_BYTE_ROUNDTRIPPABLE_CHARACTERS * - ONE_BYTE_ROUNDTRIPPABLE_CHARACTERS * - ONE_BYTE_ROUNDTRIPPABLE_CHARACTERS + + 3 * TWO_BYTE_ROUNDTRIPPABLE_CHARACTERS * ONE_BYTE_ROUNDTRIPPABLE_CHARACTERS + * ONE_BYTE_ROUNDTRIPPABLE_CHARACTERS + + // Four byte characters FOUR_BYTE_ROUNDTRIPPABLE_CHARACTERS; - static class Shard { + static final class Shard { final long index; final long start; final long lim; @@ -138,7 +183,7 @@ class IsValidUtf8TestUtil { // 97-111 are all 2342912 for (int i = 97; i <= 111; i++) { - expected[i] = 2342912; + expected[i] = 2342912; } // 113-117 are all 1048576 @@ -158,22 +203,18 @@ class IsValidUtf8TestUtil { return expected; } - static final List<Shard> FOUR_BYTE_SHARDS = generateFourByteShards( - 128, FOUR_BYTE_SHARDS_EXPECTED_ROUNTRIPPABLES); + static final List<Shard> FOUR_BYTE_SHARDS = + generateFourByteShards(128, FOUR_BYTE_SHARDS_EXPECTED_ROUNTRIPPABLES); - private static List<Shard> generateFourByteShards( - int numShards, long[] expected) { + private static List<Shard> generateFourByteShards(int numShards, long[] expected) { assertEquals(numShards, expected.length); List<Shard> shards = new ArrayList<Shard>(numShards); long LIM = 1L << 32; long increment = LIM / numShards; assertTrue(LIM % numShards == 0); for (int i = 0; i < numShards; i++) { - shards.add(new Shard(i, - increment * i, - increment * (i + 1), - expected[i])); + shards.add(new Shard(i, increment * i, increment * (i + 1), expected[i])); } return shards; } @@ -182,12 +223,12 @@ class IsValidUtf8TestUtil { * Helper to run the loop to test all the permutations for the number of bytes * specified. * + * @param factory the factory for {@link ByteString} instances. * @param numBytes the number of bytes in the byte array * @param expectedCount the expected number of roundtrippable permutations */ - static void testBytes(int numBytes, long expectedCount) - throws UnsupportedEncodingException { - testBytes(numBytes, expectedCount, 0, -1); + static void testBytes(ByteStringFactory factory, int numBytes, long expectedCount) { + testBytes(factory, numBytes, expectedCount, 0, -1); } /** @@ -195,14 +236,15 @@ class IsValidUtf8TestUtil { * specified. This overload is useful for debugging to get the loop to start * at a certain character. * + * @param factory the factory for {@link ByteString} instances. * @param numBytes the number of bytes in the byte array * @param expectedCount the expected number of roundtrippable permutations * @param start the starting bytes encoded as a long as big-endian * @param lim the limit of bytes to process encoded as a long as big-endian, * or -1 to mean the max limit for numBytes */ - static void testBytes(int numBytes, long expectedCount, long start, long lim) - throws UnsupportedEncodingException { + static void testBytes( + ByteStringFactory factory, int numBytes, long expectedCount, long start, long lim) { Random rnd = new Random(); byte[] bytes = new byte[numBytes]; @@ -217,7 +259,7 @@ class IsValidUtf8TestUtil { bytes[bytes.length - i - 1] = (byte) tmpByteChar; tmpByteChar = tmpByteChar >> 8; } - ByteString bs = ByteString.copyFrom(bytes); + ByteString bs = factory.newByteString(bytes); boolean isRoundTrippable = bs.isValidUtf8(); String s = new String(bytes, Internal.UTF_8); byte[] bytesReencoded = s.getBytes(Internal.UTF_8); @@ -236,14 +278,15 @@ class IsValidUtf8TestUtil { int i = rnd.nextInt(numBytes); int j = rnd.nextInt(numBytes); if (j < i) { - int tmp = i; i = j; j = tmp; + int tmp = i; + i = j; + j = tmp; } int state1 = Utf8.partialIsValidUtf8(Utf8.COMPLETE, bytes, 0, i); int state2 = Utf8.partialIsValidUtf8(state1, bytes, i, j); int state3 = Utf8.partialIsValidUtf8(state2, bytes, j, numBytes); if (isRoundTrippable != (state3 == Utf8.COMPLETE)) { - System.out.printf("state=%04x %04x %04x i=%d j=%d%n", - state1, state2, state3, i, j); + System.out.printf("state=%04x %04x %04x i=%d j=%d%n", state1, state2, state3, i, j); outputFailure(byteChar, bytes, bytesReencoded); } assertEquals(isRoundTrippable, (state3 == Utf8.COMPLETE)); @@ -251,36 +294,24 @@ class IsValidUtf8TestUtil { // Test ropes built out of small partial sequences ByteString rope = RopeByteString.newInstanceForTest( bs.substring(0, i), - RopeByteString.newInstanceForTest( - bs.substring(i, j), - bs.substring(j, numBytes))); + RopeByteString.newInstanceForTest(bs.substring(i, j), bs.substring(j, numBytes))); assertSame(RopeByteString.class, rope.getClass()); - ByteString[] byteStrings = { bs, bs.substring(0, numBytes), rope }; + ByteString[] byteStrings = {bs, bs.substring(0, numBytes), rope}; for (ByteString x : byteStrings) { - assertEquals(isRoundTrippable, - x.isValidUtf8()); - assertEquals(state3, - x.partialIsValidUtf8(Utf8.COMPLETE, 0, numBytes)); - - assertEquals(state1, - x.partialIsValidUtf8(Utf8.COMPLETE, 0, i)); - assertEquals(state1, - x.substring(0, i).partialIsValidUtf8(Utf8.COMPLETE, 0, i)); - assertEquals(state2, - x.partialIsValidUtf8(state1, i, j - i)); - assertEquals(state2, - x.substring(i, j).partialIsValidUtf8(state1, 0, j - i)); - assertEquals(state3, - x.partialIsValidUtf8(state2, j, numBytes - j)); - assertEquals(state3, - x.substring(j, numBytes) - .partialIsValidUtf8(state2, 0, numBytes - j)); + assertEquals(isRoundTrippable, x.isValidUtf8()); + assertEquals(state3, x.partialIsValidUtf8(Utf8.COMPLETE, 0, numBytes)); + + assertEquals(state1, x.partialIsValidUtf8(Utf8.COMPLETE, 0, i)); + assertEquals(state1, x.substring(0, i).partialIsValidUtf8(Utf8.COMPLETE, 0, i)); + assertEquals(state2, x.partialIsValidUtf8(state1, i, j - i)); + assertEquals(state2, x.substring(i, j).partialIsValidUtf8(state1, 0, j - i)); + assertEquals(state3, x.partialIsValidUtf8(state2, j, numBytes - j)); + assertEquals(state3, x.substring(j, numBytes).partialIsValidUtf8(state2, 0, numBytes - j)); } // ByteString reduplication should not affect its UTF-8 validity. - ByteString ropeADope = - RopeByteString.newInstanceForTest(bs, bs.substring(0, numBytes)); + ByteString ropeADope = RopeByteString.newInstanceForTest(bs, bs.substring(0, numBytes)); assertEquals(isRoundTrippable, ropeADope.isValidUtf8()); if (isRoundTrippable) { @@ -288,8 +319,7 @@ class IsValidUtf8TestUtil { } count++; if (byteChar != 0 && byteChar % 1000000L == 0) { - logger.info("Processed " + (byteChar / 1000000L) + - " million characters"); + logger.info("Processed " + (byteChar / 1000000L) + " million characters"); } } logger.info("Round tripped " + countRoundTripped + " of " + count); @@ -303,25 +333,26 @@ class IsValidUtf8TestUtil { * actual String class, it's possible for incompatibilities to develop * (although unlikely). * + * @param factory the factory for {@link ByteString} instances. * @param numBytes the number of bytes in the byte array * @param expectedCount the expected number of roundtrippable permutations * @param start the starting bytes encoded as a long as big-endian * @param lim the limit of bytes to process encoded as a long as big-endian, * or -1 to mean the max limit for numBytes */ - void testBytesUsingByteBuffers( - int numBytes, long expectedCount, long start, long lim) - throws UnsupportedEncodingException { - CharsetDecoder decoder = Internal.UTF_8.newDecoder() - .onMalformedInput(CodingErrorAction.REPLACE) - .onUnmappableCharacter(CodingErrorAction.REPLACE); - CharsetEncoder encoder = Internal.UTF_8.newEncoder() - .onMalformedInput(CodingErrorAction.REPLACE) - .onUnmappableCharacter(CodingErrorAction.REPLACE); + static void testBytesUsingByteBuffers( + ByteStringFactory factory, int numBytes, long expectedCount, long start, long lim) { + CharsetDecoder decoder = + Internal.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPLACE) + .onUnmappableCharacter(CodingErrorAction.REPLACE); + CharsetEncoder encoder = + Internal.UTF_8.newEncoder() + .onMalformedInput(CodingErrorAction.REPLACE) + .onUnmappableCharacter(CodingErrorAction.REPLACE); byte[] bytes = new byte[numBytes]; int maxChars = (int) (decoder.maxCharsPerByte() * numBytes) + 1; - char[] charsDecoded = - new char[(int) (decoder.maxCharsPerByte() * numBytes) + 1]; + char[] charsDecoded = new char[(int) (decoder.maxCharsPerByte() * numBytes) + 1]; int maxBytes = (int) (encoder.maxBytesPerChar() * maxChars) + 1; byte[] bytesReencoded = new byte[maxBytes]; @@ -347,7 +378,7 @@ class IsValidUtf8TestUtil { bytes[bytes.length - i - 1] = (byte) tmpByteChar; tmpByteChar = tmpByteChar >> 8; } - boolean isRoundTrippable = ByteString.copyFrom(bytes).isValidUtf8(); + boolean isRoundTrippable = factory.newByteString(bytes).isValidUtf8(); CoderResult result = decoder.decode(bb, cb, true); assertFalse(result.isError()); result = decoder.flush(cb); @@ -382,8 +413,7 @@ class IsValidUtf8TestUtil { countRoundTripped++; } if (byteChar != 0 && byteChar % 1000000 == 0) { - logger.info("Processed " + (byteChar / 1000000) + - " million characters"); + logger.info("Processed " + (byteChar / 1000000) + " million characters"); } } logger.info("Round tripped " + countRoundTripped + " of " + count); @@ -394,10 +424,9 @@ class IsValidUtf8TestUtil { outputFailure(byteChar, bytes, after, after.length); } - private static void outputFailure(long byteChar, byte[] bytes, byte[] after, - int len) { - fail("Failure: (" + Long.toHexString(byteChar) + ") " + - toHexString(bytes) + " => " + toHexString(after, len)); + private static void outputFailure(long byteChar, byte[] bytes, byte[] after, int len) { + fail("Failure: (" + Long.toHexString(byteChar) + ") " + toHexString(bytes) + " => " + + toHexString(after, len)); } private static String toHexString(byte[] b) { @@ -416,5 +445,4 @@ class IsValidUtf8TestUtil { s.append("\""); return s.toString(); } - } diff --git a/java/core/src/test/java/com/google/protobuf/LazyFieldLiteTest.java b/java/core/src/test/java/com/google/protobuf/LazyFieldLiteTest.java index 211b5697..8d1de6dc 100644 --- a/java/core/src/test/java/com/google/protobuf/LazyFieldLiteTest.java +++ b/java/core/src/test/java/com/google/protobuf/LazyFieldLiteTest.java @@ -36,9 +36,10 @@ import static protobuf_unittest.UnittestProto.optionalInt64Extension; import protobuf_unittest.UnittestProto.TestAllExtensions; import protobuf_unittest.UnittestProto.TestAllTypes; -import java.io.IOException; import junit.framework.TestCase; +import java.io.IOException; + /** * Unit test for {@link LazyFieldLite}. * diff --git a/java/core/src/test/java/com/google/protobuf/LazyFieldTest.java b/java/core/src/test/java/com/google/protobuf/LazyFieldTest.java index 2b900065..ce473ae9 100644 --- a/java/core/src/test/java/com/google/protobuf/LazyFieldTest.java +++ b/java/core/src/test/java/com/google/protobuf/LazyFieldTest.java @@ -33,9 +33,10 @@ package com.google.protobuf; import protobuf_unittest.UnittestProto.TestAllExtensions; import protobuf_unittest.UnittestProto.TestAllTypes; -import java.io.IOException; import junit.framework.TestCase; +import java.io.IOException; + /** * Unit test for {@link LazyField}. * diff --git a/java/core/src/test/java/com/google/protobuf/LiteEqualsAndHashTest.java b/java/core/src/test/java/com/google/protobuf/LiteEqualsAndHashTest.java index 035917c8..29e8d875 100644 --- a/java/core/src/test/java/com/google/protobuf/LiteEqualsAndHashTest.java +++ b/java/core/src/test/java/com/google/protobuf/LiteEqualsAndHashTest.java @@ -33,6 +33,8 @@ package com.google.protobuf; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.Bar; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.BarPrime; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.Foo; +import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.TestOneofEquals; +import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.TestRecursiveOneof; import junit.framework.TestCase; @@ -83,6 +85,16 @@ public class LiteEqualsAndHashTest extends TestCase { assertFalse(bar.equals(barPrime)); } + public void testOneofEquals() throws Exception { + TestOneofEquals.Builder builder = TestOneofEquals.newBuilder(); + TestOneofEquals message1 = builder.build(); + // Set message2's name field to default value. The two messages should be different when we + // check with the oneof case. + builder.setName(""); + TestOneofEquals message2 = builder.build(); + assertFalse(message1.equals(message2)); + } + public void testEqualsAndHashCodeWithUnknownFields() throws InvalidProtocolBufferException { Foo fooWithOnlyValue = Foo.newBuilder() .setValue(1) @@ -105,4 +117,9 @@ public class LiteEqualsAndHashTest extends TestCase { assertFalse(o1.equals(o2)); assertFalse(o1.hashCode() == o2.hashCode()); } + + public void testRecursiveHashcode() { + // This tests that we don't infinite loop. + TestRecursiveOneof.getDefaultInstance().hashCode(); + } } diff --git a/java/core/src/test/java/com/google/protobuf/LiteTest.java b/java/core/src/test/java/com/google/protobuf/LiteTest.java index b1f298ff..9e503cc3 100644 --- a/java/core/src/test/java/com/google/protobuf/LiteTest.java +++ b/java/core/src/test/java/com/google/protobuf/LiteTest.java @@ -49,6 +49,7 @@ import junit.framework.TestCase; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.NotSerializableException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; @@ -128,33 +129,7 @@ public class LiteTest extends TestCase { assertEquals(7, message2.getExtension( UnittestLite.optionalNestedMessageExtensionLite).getBb()); } - - public void testSerialize() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - TestAllTypesLite expected = - TestAllTypesLite.newBuilder() - .setOptionalInt32(123) - .addRepeatedString("hello") - .setOptionalNestedMessage( - TestAllTypesLite.NestedMessage.newBuilder().setBb(7)) - .build(); - ObjectOutputStream out = new ObjectOutputStream(baos); - try { - out.writeObject(expected); - } finally { - out.close(); - } - ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); - ObjectInputStream in = new ObjectInputStream(bais); - TestAllTypesLite actual = (TestAllTypesLite) in.readObject(); - assertEquals(expected.getOptionalInt32(), actual.getOptionalInt32()); - assertEquals(expected.getRepeatedStringCount(), - actual.getRepeatedStringCount()); - assertEquals(expected.getRepeatedString(0), - actual.getRepeatedString(0)); - assertEquals(expected.getOptionalNestedMessage().getBb(), - actual.getOptionalNestedMessage().getBb()); - } + public void testClone() { TestAllTypesLite.Builder expected = TestAllTypesLite.newBuilder() @@ -1459,4 +1434,168 @@ public class LiteTest extends TestCase { 11, (int) extendableMessage.getExtension( UnittestLite.optionalFixed32ExtensionLite)); } + + public void testToStringDefaultInstance() throws Exception { + assertToStringEquals("", TestAllTypesLite.getDefaultInstance()); + } + + public void testToStringPrimitives() throws Exception { + TestAllTypesLite proto = TestAllTypesLite.newBuilder() + .setOptionalInt32(1) + .setOptionalInt64(9223372036854775807L) + .build(); + assertToStringEquals("optional_int32: 1\noptional_int64: 9223372036854775807", proto); + + proto = TestAllTypesLite.newBuilder() + .setOptionalBool(true) + .setOptionalNestedEnum(TestAllTypesLite.NestedEnum.BAZ) + .build(); + assertToStringEquals("optional_bool: true\noptional_nested_enum: BAZ", proto); + + proto = TestAllTypesLite.newBuilder() + .setOptionalFloat(2.72f) + .setOptionalDouble(3.14) + .build(); + assertToStringEquals("optional_float: 2.72\noptional_double: 3.14", proto); + } + + public void testToStringStringFields() throws Exception { + TestAllTypesLite proto = TestAllTypesLite.newBuilder() + .setOptionalString("foo\"bar\nbaz\\") + .build(); + assertToStringEquals("optional_string: \"foo\\\"bar\\nbaz\\\\\"", proto); + + proto = TestAllTypesLite.newBuilder() + .setOptionalString("\u6587") + .build(); + assertToStringEquals("optional_string: \"\\346\\226\\207\"", proto); + } + + public void testToStringNestedMessage() throws Exception { + TestAllTypesLite proto = TestAllTypesLite.newBuilder() + .setOptionalNestedMessage(TestAllTypesLite.NestedMessage.getDefaultInstance()) + .build(); + assertToStringEquals("optional_nested_message {\n}", proto); + + proto = TestAllTypesLite.newBuilder() + .setOptionalNestedMessage( + TestAllTypesLite.NestedMessage.newBuilder().setBb(7)) + .build(); + assertToStringEquals("optional_nested_message {\n bb: 7\n}", proto); + } + + public void testToStringRepeatedFields() throws Exception { + TestAllTypesLite proto = TestAllTypesLite.newBuilder() + .addRepeatedInt32(32) + .addRepeatedInt32(32) + .addRepeatedInt64(64) + .build(); + assertToStringEquals("repeated_int32: 32\nrepeated_int32: 32\nrepeated_int64: 64", proto); + + proto = TestAllTypesLite.newBuilder() + .addRepeatedLazyMessage( + TestAllTypesLite.NestedMessage.newBuilder().setBb(7)) + .addRepeatedLazyMessage( + TestAllTypesLite.NestedMessage.newBuilder().setBb(8)) + .build(); + assertToStringEquals( + "repeated_lazy_message {\n bb: 7\n}\nrepeated_lazy_message {\n bb: 8\n}", + proto); + } + + public void testToStringForeignFields() throws Exception { + TestAllTypesLite proto = TestAllTypesLite.newBuilder() + .setOptionalForeignEnum(ForeignEnumLite.FOREIGN_LITE_BAR) + .setOptionalForeignMessage( + ForeignMessageLite.newBuilder() + .setC(3)) + .build(); + assertToStringEquals( + "optional_foreign_message {\n c: 3\n}\noptional_foreign_enum: FOREIGN_LITE_BAR", + proto); + } + + public void testToStringExtensions() throws Exception { + TestAllExtensionsLite message = TestAllExtensionsLite.newBuilder() + .setExtension(UnittestLite.optionalInt32ExtensionLite, 123) + .addExtension(UnittestLite.repeatedStringExtensionLite, "spam") + .addExtension(UnittestLite.repeatedStringExtensionLite, "eggs") + .setExtension(UnittestLite.optionalNestedEnumExtensionLite, + TestAllTypesLite.NestedEnum.BAZ) + .setExtension(UnittestLite.optionalNestedMessageExtensionLite, + TestAllTypesLite.NestedMessage.newBuilder().setBb(7).build()) + .build(); + assertToStringEquals( + "[1]: 123\n[18] {\n bb: 7\n}\n[21]: 3\n[44]: \"spam\"\n[44]: \"eggs\"", + message); + } + + public void testToStringUnknownFields() throws Exception { + TestAllExtensionsLite messageWithExtensions = TestAllExtensionsLite.newBuilder() + .setExtension(UnittestLite.optionalInt32ExtensionLite, 123) + .addExtension(UnittestLite.repeatedStringExtensionLite, "spam") + .addExtension(UnittestLite.repeatedStringExtensionLite, "eggs") + .setExtension(UnittestLite.optionalNestedEnumExtensionLite, + TestAllTypesLite.NestedEnum.BAZ) + .setExtension(UnittestLite.optionalNestedMessageExtensionLite, + TestAllTypesLite.NestedMessage.newBuilder().setBb(7).build()) + .build(); + TestAllExtensionsLite messageWithUnknownFields = TestAllExtensionsLite.parseFrom( + messageWithExtensions.toByteArray()); + assertToStringEquals( + "1: 123\n18: \"\\b\\a\"\n21: 3\n44: \"spam\"\n44: \"eggs\"", + messageWithUnknownFields); + } + + // Asserts that the toString() representation of the message matches the expected. This verifies + // the first line starts with a comment; but, does not factor in said comment as part of the + // comparison as it contains unstable addresses. + private static void assertToStringEquals(String expected, MessageLite message) { + String toString = message.toString(); + assertEquals('#', toString.charAt(0)); + if (toString.indexOf("\n") >= 0) { + toString = toString.substring(toString.indexOf("\n") + 1); + } else { + toString = ""; + } + assertEquals(expected, toString); + } + + public void testParseLazy() throws Exception { + ByteString bb = TestAllTypesLite.newBuilder() + .setOptionalLazyMessage(NestedMessage.newBuilder() + .setBb(11) + .build()) + .build().toByteString(); + ByteString cc = TestAllTypesLite.newBuilder() + .setOptionalLazyMessage(NestedMessage.newBuilder() + .setCc(22) + .build()) + .build().toByteString(); + + ByteString concat = bb.concat(cc); + TestAllTypesLite message = TestAllTypesLite.parseFrom(concat); + + assertEquals(11, message.getOptionalLazyMessage().getBb()); + assertEquals(22L, message.getOptionalLazyMessage().getCc()); + } + + public void testParseLazy_oneOf() throws Exception { + ByteString bb = TestAllTypesLite.newBuilder() + .setOneofLazyNestedMessage(NestedMessage.newBuilder() + .setBb(11) + .build()) + .build().toByteString(); + ByteString cc = TestAllTypesLite.newBuilder() + .setOneofLazyNestedMessage(NestedMessage.newBuilder() + .setCc(22) + .build()) + .build().toByteString(); + + ByteString concat = bb.concat(cc); + TestAllTypesLite message = TestAllTypesLite.parseFrom(concat); + + assertEquals(11, message.getOneofLazyNestedMessage().getBb()); + assertEquals(22L, message.getOneofLazyNestedMessage().getCc()); + } } diff --git a/java/core/src/test/java/com/google/protobuf/LiteralByteStringTest.java b/java/core/src/test/java/com/google/protobuf/LiteralByteStringTest.java index 68b55ceb..2e7792a8 100644 --- a/java/core/src/test/java/com/google/protobuf/LiteralByteStringTest.java +++ b/java/core/src/test/java/com/google/protobuf/LiteralByteStringTest.java @@ -47,9 +47,9 @@ import java.util.List; import java.util.NoSuchElementException; /** - * Test {@link LiteralByteString} by setting up a reference string in {@link #setUp()}. - * This class is designed to be extended for testing extensions of {@link LiteralByteString} - * such as {@link BoundedByteString}, see {@link BoundedByteStringTest}. + * Test {@code LiteralByteString} by setting up a reference string in {@link #setUp()}. + * This class is designed to be extended for testing extensions of {@code LiteralByteString} + * such as {@code BoundedByteString}, see {@link BoundedByteStringTest}. * * @author carlanton@google.com (Carl Haverl) */ @@ -304,25 +304,75 @@ public class LiteralByteStringTest extends TestCase { Arrays.equals(referenceBytes, roundTripBytes)); } - public void testWriteTo_mutating() throws IOException { + public void testWriteToShouldNotExposeInternalBufferToOutputStream() throws IOException { OutputStream os = new OutputStream() { @Override public void write(byte[] b, int off, int len) { - for (int x = 0; x < len; ++x) { - b[off + x] = (byte) 0; - } + Arrays.fill(b, off, off + len, (byte) 0); } @Override public void write(int b) { - // Purposefully left blank. + throw new UnsupportedOperationException(); } }; stringUnderTest.writeTo(os); - byte[] newBytes = stringUnderTest.toByteArray(); assertTrue(classUnderTest + ".writeTo() must not grant access to underlying array", - Arrays.equals(referenceBytes, newBytes)); + Arrays.equals(referenceBytes, stringUnderTest.toByteArray())); + } + + public void testWriteToInternalShouldExposeInternalBufferToOutputStream() throws IOException { + OutputStream os = new OutputStream() { + @Override + public void write(byte[] b, int off, int len) { + Arrays.fill(b, off, off + len, (byte) 0); + } + + @Override + public void write(int b) { + throw new UnsupportedOperationException(); + } + }; + + stringUnderTest.writeToInternal(os, 0, stringUnderTest.size()); + byte[] allZeros = new byte[stringUnderTest.size()]; + assertTrue(classUnderTest + ".writeToInternal() must grant access to underlying array", + Arrays.equals(allZeros, stringUnderTest.toByteArray())); + } + + public void testWriteToShouldExposeInternalBufferToByteOutput() throws IOException { + ByteOutput out = new ByteOutput() { + @Override + public void write(byte value) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void write(byte[] value, int offset, int length) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void writeLazy(byte[] value, int offset, int length) throws IOException { + Arrays.fill(value, offset, offset + length, (byte) 0); + } + + @Override + public void write(ByteBuffer value) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void writeLazy(ByteBuffer value) throws IOException { + throw new UnsupportedOperationException(); + } + }; + + stringUnderTest.writeTo(out); + byte[] allZeros = new byte[stringUnderTest.size()]; + assertTrue(classUnderTest + ".writeToInternal() must grant access to underlying array", + Arrays.equals(allZeros, stringUnderTest.toByteArray())); } public void testNewOutput() throws IOException { diff --git a/java/core/src/test/java/com/google/protobuf/MapForProto2LiteTest.java b/java/core/src/test/java/com/google/protobuf/MapForProto2LiteTest.java index 3d8c9bc4..d79d0029 100644 --- a/java/core/src/test/java/com/google/protobuf/MapForProto2LiteTest.java +++ b/java/core/src/test/java/com/google/protobuf/MapForProto2LiteTest.java @@ -432,7 +432,7 @@ public class MapForProto2LiteTest extends TestCase { assertEquals(1, messageWithUnknownEnums.getInt32ToInt32Field().get(1).intValue()); assertEquals(54321, messageWithUnknownEnums.getInt32ToInt32Field().get(2).intValue()); } - + public void testIterationOrder() throws Exception { TestMap.Builder builder = TestMap.newBuilder(); diff --git a/java/core/src/test/java/com/google/protobuf/MapForProto2Test.java b/java/core/src/test/java/com/google/protobuf/MapForProto2Test.java index 1fa3cbdb..73154c0f 100644 --- a/java/core/src/test/java/com/google/protobuf/MapForProto2Test.java +++ b/java/core/src/test/java/com/google/protobuf/MapForProto2Test.java @@ -36,7 +36,6 @@ import map_test.MapForProto2TestProto.TestMap.MessageValue; import map_test.MapForProto2TestProto.TestMap.MessageWithRequiredFields; import map_test.MapForProto2TestProto.TestRecursiveMap; import map_test.MapForProto2TestProto.TestUnknownEnumValue; - import junit.framework.TestCase; import java.util.ArrayList; diff --git a/java/core/src/test/java/com/google/protobuf/MapTest.java b/java/core/src/test/java/com/google/protobuf/MapTest.java index 0e5c1284..1dc5787d 100644 --- a/java/core/src/test/java/com/google/protobuf/MapTest.java +++ b/java/core/src/test/java/com/google/protobuf/MapTest.java @@ -37,7 +37,6 @@ import com.google.protobuf.Descriptors.FieldDescriptor; import map_test.MapTestProto.TestMap; import map_test.MapTestProto.TestMap.MessageValue; import map_test.MapTestProto.TestOnChangeEventPropagation; - import junit.framework.TestCase; import java.util.ArrayList; diff --git a/java/core/src/test/java/com/google/protobuf/MessageTest.java b/java/core/src/test/java/com/google/protobuf/MessageTest.java index abcd3a1d..dcd1aba7 100644 --- a/java/core/src/test/java/com/google/protobuf/MessageTest.java +++ b/java/core/src/test/java/com/google/protobuf/MessageTest.java @@ -30,11 +30,11 @@ package com.google.protobuf; -import protobuf_unittest.UnittestProto.TestAllTypes; +import protobuf_unittest.UnittestProto.ForeignMessage; import protobuf_unittest.UnittestProto.TestAllExtensions; +import protobuf_unittest.UnittestProto.TestAllTypes; import protobuf_unittest.UnittestProto.TestRequired; import protobuf_unittest.UnittestProto.TestRequiredForeign; -import protobuf_unittest.UnittestProto.ForeignMessage; import junit.framework.TestCase; diff --git a/java/core/src/test/java/com/google/protobuf/NestedBuildersTest.java b/java/core/src/test/java/com/google/protobuf/NestedBuildersTest.java index 23653126..542e28c0 100644 --- a/java/core/src/test/java/com/google/protobuf/NestedBuildersTest.java +++ b/java/core/src/test/java/com/google/protobuf/NestedBuildersTest.java @@ -35,8 +35,8 @@ import protobuf_unittest.Wheel; import junit.framework.TestCase; -import java.util.List; import java.util.ArrayList; +import java.util.List; /** * Test cases that exercise end-to-end use cases involving diff --git a/java/core/src/test/java/com/google/protobuf/NioByteStringTest.java b/java/core/src/test/java/com/google/protobuf/NioByteStringTest.java index e40a3662..6be5b93c 100644 --- a/java/core/src/test/java/com/google/protobuf/NioByteStringTest.java +++ b/java/core/src/test/java/com/google/protobuf/NioByteStringTest.java @@ -41,6 +41,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; +import java.io.OutputStream; import java.io.UnsupportedEncodingException; import java.nio.BufferOverflowException; import java.nio.ByteBuffer; @@ -56,11 +57,12 @@ public class NioByteStringTest extends TestCase { private static final String CLASSNAME = NioByteString.class.getSimpleName(); private static final byte[] BYTES = ByteStringTest.getTestBytes(1234, 11337766L); private static final int EXPECTED_HASH = ByteString.wrap(BYTES).hashCode(); - private static final ByteBuffer BUFFER = ByteBuffer.wrap(BYTES.clone()); - private static final ByteString TEST_STRING = new NioByteString(BUFFER); + + private final ByteBuffer backingBuffer = ByteBuffer.wrap(BYTES.clone()); + private final ByteString testString = new NioByteString(backingBuffer); public void testExpectedType() { - String actualClassName = getActualClassName(TEST_STRING); + String actualClassName = getActualClassName(testString); assertEquals(CLASSNAME + " should match type exactly", CLASSNAME, actualClassName); } @@ -73,14 +75,14 @@ public class NioByteStringTest extends TestCase { public void testByteAt() { boolean stillEqual = true; for (int i = 0; stillEqual && i < BYTES.length; ++i) { - stillEqual = (BYTES[i] == TEST_STRING.byteAt(i)); + stillEqual = (BYTES[i] == testString.byteAt(i)); } assertTrue(CLASSNAME + " must capture the right bytes", stillEqual); } public void testByteIterator() { boolean stillEqual = true; - ByteString.ByteIterator iter = TEST_STRING.iterator(); + ByteString.ByteIterator iter = testString.iterator(); for (int i = 0; stillEqual && i < BYTES.length; ++i) { stillEqual = (iter.hasNext() && BYTES[i] == iter.nextByte()); } @@ -98,7 +100,7 @@ public class NioByteStringTest extends TestCase { public void testByteIterable() { boolean stillEqual = true; int j = 0; - for (byte quantum : TEST_STRING) { + for (byte quantum : testString) { stillEqual = (BYTES[j] == quantum); ++j; } @@ -108,15 +110,15 @@ public class NioByteStringTest extends TestCase { public void testSize() { assertEquals(CLASSNAME + " must have the expected size", BYTES.length, - TEST_STRING.size()); + testString.size()); } public void testGetTreeDepth() { - assertEquals(CLASSNAME + " must have depth 0", 0, TEST_STRING.getTreeDepth()); + assertEquals(CLASSNAME + " must have depth 0", 0, testString.getTreeDepth()); } public void testIsBalanced() { - assertTrue(CLASSNAME + " is technically balanced", TEST_STRING.isBalanced()); + assertTrue(CLASSNAME + " is technically balanced", testString.isBalanced()); } public void testCopyTo_ByteArrayOffsetLength() { @@ -124,7 +126,7 @@ public class NioByteStringTest extends TestCase { int length = 100; byte[] destination = new byte[destinationOffset + length]; int sourceOffset = 213; - TEST_STRING.copyTo(destination, sourceOffset, destinationOffset, length); + testString.copyTo(destination, sourceOffset, destinationOffset, length); boolean stillEqual = true; for (int i = 0; stillEqual && i < length; ++i) { stillEqual = BYTES[i + sourceOffset] == destination[i + destinationOffset]; @@ -139,7 +141,7 @@ public class NioByteStringTest extends TestCase { try { // Copy one too many bytes - TEST_STRING.copyTo(destination, TEST_STRING.size() + 1 - length, + testString.copyTo(destination, testString.size() + 1 - length, destinationOffset, length); fail("Should have thrown an exception when copying too many bytes of a " + CLASSNAME); @@ -149,7 +151,7 @@ public class NioByteStringTest extends TestCase { try { // Copy with illegal negative sourceOffset - TEST_STRING.copyTo(destination, -1, destinationOffset, length); + testString.copyTo(destination, -1, destinationOffset, length); fail("Should have thrown an exception when given a negative sourceOffset in " + CLASSNAME); } catch (IndexOutOfBoundsException expected) { @@ -158,7 +160,7 @@ public class NioByteStringTest extends TestCase { try { // Copy with illegal negative destinationOffset - TEST_STRING.copyTo(destination, 0, -1, length); + testString.copyTo(destination, 0, -1, length); fail("Should have thrown an exception when given a negative destinationOffset in " + CLASSNAME); } catch (IndexOutOfBoundsException expected) { @@ -167,7 +169,7 @@ public class NioByteStringTest extends TestCase { try { // Copy with illegal negative size - TEST_STRING.copyTo(destination, 0, 0, -1); + testString.copyTo(destination, 0, 0, -1); fail("Should have thrown an exception when given a negative size in " + CLASSNAME); } catch (IndexOutOfBoundsException expected) { @@ -176,7 +178,7 @@ public class NioByteStringTest extends TestCase { try { // Copy with illegal too-large sourceOffset - TEST_STRING.copyTo(destination, 2 * TEST_STRING.size(), 0, length); + testString.copyTo(destination, 2 * testString.size(), 0, length); fail("Should have thrown an exception when the destinationOffset is too large in " + CLASSNAME); } catch (IndexOutOfBoundsException expected) { @@ -185,7 +187,7 @@ public class NioByteStringTest extends TestCase { try { // Copy with illegal too-large destinationOffset - TEST_STRING.copyTo(destination, 0, 2 * destination.length, length); + testString.copyTo(destination, 0, 2 * destination.length, length); fail("Should have thrown an exception when the destinationOffset is too large in " + CLASSNAME); } catch (IndexOutOfBoundsException expected) { @@ -196,21 +198,21 @@ public class NioByteStringTest extends TestCase { public void testCopyTo_ByteBuffer() { // Same length. ByteBuffer myBuffer = ByteBuffer.allocate(BYTES.length); - TEST_STRING.copyTo(myBuffer); + testString.copyTo(myBuffer); myBuffer.flip(); assertEquals(CLASSNAME + ".copyTo(ByteBuffer) must give back the same bytes", - BUFFER, myBuffer); + backingBuffer, myBuffer); // Target buffer bigger than required. - myBuffer = ByteBuffer.allocate(TEST_STRING.size() + 1); - TEST_STRING.copyTo(myBuffer); + myBuffer = ByteBuffer.allocate(testString.size() + 1); + testString.copyTo(myBuffer); myBuffer.flip(); - assertEquals(BUFFER, myBuffer); + assertEquals(backingBuffer, myBuffer); // Target buffer has no space. myBuffer = ByteBuffer.allocate(0); try { - TEST_STRING.copyTo(myBuffer); + testString.copyTo(myBuffer); fail("Should have thrown an exception when target ByteBuffer has insufficient capacity"); } catch (BufferOverflowException e) { // Expected. @@ -219,7 +221,7 @@ public class NioByteStringTest extends TestCase { // Target buffer too small. myBuffer = ByteBuffer.allocate(1); try { - TEST_STRING.copyTo(myBuffer); + testString.copyTo(myBuffer); fail("Should have thrown an exception when target ByteBuffer has insufficient capacity"); } catch (BufferOverflowException e) { // Expected. @@ -227,26 +229,26 @@ public class NioByteStringTest extends TestCase { } public void testMarkSupported() { - InputStream stream = TEST_STRING.newInput(); + InputStream stream = testString.newInput(); assertTrue(CLASSNAME + ".newInput() must support marking", stream.markSupported()); } public void testMarkAndReset() throws IOException { - int fraction = TEST_STRING.size() / 3; + int fraction = testString.size() / 3; - InputStream stream = TEST_STRING.newInput(); - stream.mark(TEST_STRING.size()); // First, mark() the end. + InputStream stream = testString.newInput(); + stream.mark(testString.size()); // First, mark() the end. skipFully(stream, fraction); // Skip a large fraction, but not all. assertEquals( CLASSNAME + ": after skipping to the 'middle', half the bytes are available", - (TEST_STRING.size() - fraction), stream.available()); + (testString.size() - fraction), stream.available()); stream.reset(); assertEquals( CLASSNAME + ": after resetting, all bytes are available", - TEST_STRING.size(), stream.available()); + testString.size(), stream.available()); - skipFully(stream, TEST_STRING.size()); // Skip to the end. + skipFully(stream, testString.size()); // Skip to the end. assertEquals( CLASSNAME + ": after skipping to the end, no more bytes are available", 0, stream.available()); @@ -284,7 +286,7 @@ public class NioByteStringTest extends TestCase { } public void testAsReadOnlyByteBuffer() { - ByteBuffer byteBuffer = TEST_STRING.asReadOnlyByteBuffer(); + ByteBuffer byteBuffer = testString.asReadOnlyByteBuffer(); byte[] roundTripBytes = new byte[BYTES.length]; assertTrue(byteBuffer.remaining() == BYTES.length); assertTrue(byteBuffer.isReadOnly()); @@ -294,7 +296,7 @@ public class NioByteStringTest extends TestCase { } public void testAsReadOnlyByteBufferList() { - List<ByteBuffer> byteBuffers = TEST_STRING.asReadOnlyByteBufferList(); + List<ByteBuffer> byteBuffers = testString.asReadOnlyByteBufferList(); int bytesSeen = 0; byte[] roundTripBytes = new byte[BYTES.length]; for (ByteBuffer byteBuffer : byteBuffers) { @@ -310,25 +312,98 @@ public class NioByteStringTest extends TestCase { } public void testToByteArray() { - byte[] roundTripBytes = TEST_STRING.toByteArray(); + byte[] roundTripBytes = testString.toByteArray(); assertTrue(CLASSNAME + ".toByteArray() must give back the same bytes", Arrays.equals(BYTES, roundTripBytes)); } public void testWriteTo() throws IOException { ByteArrayOutputStream bos = new ByteArrayOutputStream(); - TEST_STRING.writeTo(bos); + testString.writeTo(bos); byte[] roundTripBytes = bos.toByteArray(); assertTrue(CLASSNAME + ".writeTo() must give back the same bytes", Arrays.equals(BYTES, roundTripBytes)); } + public void testWriteToShouldNotExposeInternalBufferToOutputStream() throws IOException { + OutputStream os = new OutputStream() { + @Override + public void write(byte[] b, int off, int len) { + Arrays.fill(b, off, off + len, (byte) 0); + } + + @Override + public void write(int b) { + throw new UnsupportedOperationException(); + } + }; + + byte[] original = Arrays.copyOf(BYTES, BYTES.length); + testString.writeTo(os); + assertTrue(CLASSNAME + ".writeTo() must NOT grant access to underlying buffer", + Arrays.equals(original, BYTES)); + } + + public void testWriteToInternalShouldExposeInternalBufferToOutputStream() throws IOException { + OutputStream os = new OutputStream() { + @Override + public void write(byte[] b, int off, int len) { + Arrays.fill(b, off, off + len, (byte) 0); + } + + @Override + public void write(int b) { + throw new UnsupportedOperationException(); + } + }; + + testString.writeToInternal(os, 0, testString.size()); + byte[] allZeros = new byte[testString.size()]; + assertTrue(CLASSNAME + ".writeToInternal() must grant access to underlying buffer", + Arrays.equals(allZeros, backingBuffer.array())); + } + + public void testWriteToShouldExposeInternalBufferToByteOutput() throws IOException { + ByteOutput out = new ByteOutput() { + @Override + public void write(byte value) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void write(byte[] value, int offset, int length) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void writeLazy(byte[] value, int offset, int length) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void write(ByteBuffer value) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void writeLazy(ByteBuffer value) throws IOException { + Arrays.fill(value.array(), value.arrayOffset(), value.arrayOffset() + value.limit(), + (byte) 0); + } + }; + + testString.writeTo(out); + byte[] allZeros = new byte[testString.size()]; + assertTrue(CLASSNAME + ".writeTo() must grant access to underlying buffer", + Arrays.equals(allZeros, backingBuffer.array())); + } + public void testNewOutput() throws IOException { ByteArrayOutputStream bos = new ByteArrayOutputStream(); ByteString.Output output = ByteString.newOutput(); - TEST_STRING.writeTo(output); + testString.writeTo(output); assertEquals("Output Size returns correct result", - output.size(), TEST_STRING.size()); + output.size(), testString.size()); output.writeTo(bos); assertTrue("Output.writeTo() must give back the same bytes", Arrays.equals(BYTES, bos.toByteArray())); @@ -336,7 +411,7 @@ public class NioByteStringTest extends TestCase { // write the output stream to itself! This should cause it to double output.writeTo(output); assertEquals("Writing an output stream to itself is successful", - TEST_STRING.concat(TEST_STRING), output.toByteString()); + testString.concat(testString), output.toByteString()); output.reset(); assertEquals("Output.reset() resets the output", 0, output.size()); @@ -373,7 +448,7 @@ public class NioByteStringTest extends TestCase { } try { - TEST_STRING.toString("invalid"); + testString.toString("invalid"); fail("Should have thrown an exception."); } catch (UnsupportedEncodingException expected) { // This is success @@ -381,36 +456,36 @@ public class NioByteStringTest extends TestCase { } public void testEquals() { - assertEquals(CLASSNAME + " must not equal null", false, TEST_STRING.equals(null)); - assertEquals(CLASSNAME + " must equal self", TEST_STRING, TEST_STRING); + assertEquals(CLASSNAME + " must not equal null", false, testString.equals(null)); + assertEquals(CLASSNAME + " must equal self", testString, testString); assertFalse(CLASSNAME + " must not equal the empty string", - TEST_STRING.equals(EMPTY)); + testString.equals(EMPTY)); assertEquals(CLASSNAME + " empty strings must be equal", - EMPTY, TEST_STRING.substring(55, 55)); + EMPTY, testString.substring(55, 55)); assertEquals(CLASSNAME + " must equal another string with the same value", - TEST_STRING, new NioByteString(BUFFER)); + testString, new NioByteString(backingBuffer)); byte[] mungedBytes = mungedBytes(); assertFalse(CLASSNAME + " must not equal every string with the same length", - TEST_STRING.equals(new NioByteString(ByteBuffer.wrap(mungedBytes)))); + testString.equals(new NioByteString(ByteBuffer.wrap(mungedBytes)))); } public void testEqualsLiteralByteString() { ByteString literal = ByteString.copyFrom(BYTES); assertEquals(CLASSNAME + " must equal LiteralByteString with same value", literal, - TEST_STRING); - assertEquals(CLASSNAME + " must equal LiteralByteString with same value", TEST_STRING, + testString); + assertEquals(CLASSNAME + " must equal LiteralByteString with same value", testString, literal); assertFalse(CLASSNAME + " must not equal the empty string", - TEST_STRING.equals(ByteString.EMPTY)); + testString.equals(ByteString.EMPTY)); assertEquals(CLASSNAME + " empty strings must be equal", - ByteString.EMPTY, TEST_STRING.substring(55, 55)); + ByteString.EMPTY, testString.substring(55, 55)); literal = ByteString.copyFrom(mungedBytes()); assertFalse(CLASSNAME + " must not equal every LiteralByteString with the same length", - TEST_STRING.equals(literal)); + testString.equals(literal)); assertFalse(CLASSNAME + " must not equal every LiteralByteString with the same length", - literal.equals(TEST_STRING)); + literal.equals(testString)); } public void testEqualsRopeByteString() { @@ -419,22 +494,22 @@ public class NioByteStringTest extends TestCase { ByteString rope = p1.concat(p2); assertEquals(CLASSNAME + " must equal RopeByteString with same value", rope, - TEST_STRING); - assertEquals(CLASSNAME + " must equal RopeByteString with same value", TEST_STRING, + testString); + assertEquals(CLASSNAME + " must equal RopeByteString with same value", testString, rope); assertFalse(CLASSNAME + " must not equal the empty string", - TEST_STRING.equals(ByteString.EMPTY.concat(ByteString.EMPTY))); + testString.equals(ByteString.EMPTY.concat(ByteString.EMPTY))); assertEquals(CLASSNAME + " empty strings must be equal", - ByteString.EMPTY.concat(ByteString.EMPTY), TEST_STRING.substring(55, 55)); + ByteString.EMPTY.concat(ByteString.EMPTY), testString.substring(55, 55)); byte[] mungedBytes = mungedBytes(); p1 = ByteString.copyFrom(mungedBytes, 0, 5); p2 = ByteString.copyFrom(mungedBytes, 5, mungedBytes.length - 5); rope = p1.concat(p2); assertFalse(CLASSNAME + " must not equal every RopeByteString with the same length", - TEST_STRING.equals(rope)); + testString.equals(rope)); assertFalse(CLASSNAME + " must not equal every RopeByteString with the same length", - rope.equals(TEST_STRING)); + rope.equals(testString)); } private byte[] mungedBytes() { @@ -445,12 +520,12 @@ public class NioByteStringTest extends TestCase { } public void testHashCode() { - int hash = TEST_STRING.hashCode(); + int hash = testString.hashCode(); assertEquals(CLASSNAME + " must have expected hashCode", EXPECTED_HASH, hash); } public void testPeekCachedHashCode() { - ByteString newString = new NioByteString(BUFFER); + ByteString newString = new NioByteString(backingBuffer); assertEquals(CLASSNAME + ".peekCachedHashCode() should return zero at first", 0, newString.peekCachedHashCode()); newString.hashCode(); @@ -461,15 +536,15 @@ public class NioByteStringTest extends TestCase { public void testPartialHash() { // partialHash() is more strenuously tested elsewhere by testing hashes of substrings. // This test would fail if the expected hash were 1. It's not. - int hash = TEST_STRING.partialHash(TEST_STRING.size(), 0, TEST_STRING.size()); + int hash = testString.partialHash(testString.size(), 0, testString.size()); assertEquals(CLASSNAME + ".partialHash() must yield expected hashCode", EXPECTED_HASH, hash); } public void testNewInput() throws IOException { - InputStream input = TEST_STRING.newInput(); + InputStream input = testString.newInput(); assertEquals("InputStream.available() returns correct value", - TEST_STRING.size(), input.available()); + testString.size(), input.available()); boolean stillEqual = true; for (byte referenceByte : BYTES) { int expectedInt = (referenceByte & 0xFF); @@ -482,8 +557,8 @@ public class NioByteStringTest extends TestCase { } public void testNewInput_skip() throws IOException { - InputStream input = TEST_STRING.newInput(); - int stringSize = TEST_STRING.size(); + InputStream input = testString.newInput(); + int stringSize = testString.size(); int nearEndIndex = stringSize * 2 / 3; long skipped1 = input.skip(nearEndIndex); assertEquals("InputStream.skip()", skipped1, nearEndIndex); @@ -492,7 +567,7 @@ public class NioByteStringTest extends TestCase { assertTrue("InputStream.mark() is available", input.markSupported()); input.mark(0); assertEquals("InputStream.skip(), read()", - TEST_STRING.byteAt(nearEndIndex) & 0xFF, input.read()); + testString.byteAt(nearEndIndex) & 0xFF, input.read()); assertEquals("InputStream.available()", stringSize - skipped1 - 1, input.available()); long skipped2 = input.skip(stringSize); @@ -504,11 +579,11 @@ public class NioByteStringTest extends TestCase { assertEquals("InputStream.reset() succeded", stringSize - skipped1, input.available()); assertEquals("InputStream.reset(), read()", - TEST_STRING.byteAt(nearEndIndex) & 0xFF, input.read()); + testString.byteAt(nearEndIndex) & 0xFF, input.read()); } public void testNewCodedInput() throws IOException { - CodedInputStream cis = TEST_STRING.newCodedInput(); + CodedInputStream cis = testString.newCodedInput(); byte[] roundTripBytes = cis.readRawBytes(BYTES.length); assertTrue(CLASSNAME + " must give the same bytes back from the CodedInputStream", Arrays.equals(BYTES, roundTripBytes)); @@ -521,22 +596,22 @@ public class NioByteStringTest extends TestCase { */ public void testConcat_empty() { assertSame(CLASSNAME + " concatenated with empty must give " + CLASSNAME, - TEST_STRING.concat(EMPTY), TEST_STRING); + testString.concat(EMPTY), testString); assertSame("empty concatenated with " + CLASSNAME + " must give " + CLASSNAME, - EMPTY.concat(TEST_STRING), TEST_STRING); + EMPTY.concat(testString), testString); } public void testJavaSerialization() throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); ObjectOutputStream oos = new ObjectOutputStream(out); - oos.writeObject(TEST_STRING); + oos.writeObject(testString); oos.close(); byte[] pickled = out.toByteArray(); InputStream in = new ByteArrayInputStream(pickled); ObjectInputStream ois = new ObjectInputStream(in); Object o = ois.readObject(); assertTrue("Didn't get a ByteString back", o instanceof ByteString); - assertEquals("Should get an equal ByteString back", TEST_STRING, o); + assertEquals("Should get an equal ByteString back", testString, o); } private static ByteString forString(String str) { diff --git a/java/core/src/test/java/com/google/protobuf/ParseExceptionsTest.java b/java/core/src/test/java/com/google/protobuf/ParseExceptionsTest.java index 37fa242d..bf1f1d71 100644 --- a/java/core/src/test/java/com/google/protobuf/ParseExceptionsTest.java +++ b/java/core/src/test/java/com/google/protobuf/ParseExceptionsTest.java @@ -1,17 +1,51 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + package com.google.protobuf; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import com.google.protobuf.DescriptorProtos.DescriptorProto; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; /** * Tests the exceptions thrown when parsing from a stream. The methods on the {@link Parser} @@ -22,6 +56,7 @@ import static org.junit.Assert.fail; * * @author jh@squareup.com (Joshua Humphries) */ +@RunWith(JUnit4.class) public class ParseExceptionsTest { private interface ParseTester { @@ -46,116 +81,143 @@ public class ParseExceptionsTest { @Test public void message_parseFrom_InputStream() { setup(); - verifyExceptions(new ParseTester() { - public DescriptorProto parse(InputStream in) throws IOException { - return DescriptorProto.parseFrom(in); - } - }); + verifyExceptions( + new ParseTester() { + @Override + public DescriptorProto parse(InputStream in) throws IOException { + return DescriptorProto.parseFrom(in); + } + }); } @Test public void message_parseFrom_InputStreamAndExtensionRegistry() { setup(); - verifyExceptions(new ParseTester() { - public DescriptorProto parse(InputStream in) throws IOException { - return DescriptorProto.parseFrom(in, ExtensionRegistry.newInstance()); - } - }); + verifyExceptions( + new ParseTester() { + @Override + public DescriptorProto parse(InputStream in) throws IOException { + return DescriptorProto.parseFrom(in, ExtensionRegistry.newInstance()); + } + }); } @Test public void message_parseFrom_CodedInputStream() { setup(); - verifyExceptions(new ParseTester() { - public DescriptorProto parse(InputStream in) throws IOException { - return DescriptorProto.parseFrom(CodedInputStream.newInstance(in)); - } - }); + verifyExceptions( + new ParseTester() { + @Override + public DescriptorProto parse(InputStream in) throws IOException { + return DescriptorProto.parseFrom(CodedInputStream.newInstance(in)); + } + }); } @Test public void message_parseFrom_CodedInputStreamAndExtensionRegistry() { setup(); - verifyExceptions(new ParseTester() { - public DescriptorProto parse(InputStream in) throws IOException { - return DescriptorProto.parseFrom(CodedInputStream.newInstance(in), - ExtensionRegistry.newInstance()); - } - }); + verifyExceptions( + new ParseTester() { + @Override + public DescriptorProto parse(InputStream in) throws IOException { + return DescriptorProto.parseFrom( + CodedInputStream.newInstance(in), ExtensionRegistry.newInstance()); + } + }); } @Test public void message_parseDelimitedFrom_InputStream() { setupDelimited(); - verifyExceptions(new ParseTester() { - public DescriptorProto parse(InputStream in) throws IOException { - return DescriptorProto.parseDelimitedFrom(in); - } - }); + verifyExceptions( + new ParseTester() { + @Override + public DescriptorProto parse(InputStream in) throws IOException { + return DescriptorProto.parseDelimitedFrom(in); + } + }); } @Test public void message_parseDelimitedFrom_InputStreamAndExtensionRegistry() { setupDelimited(); - verifyExceptions(new ParseTester() { - public DescriptorProto parse(InputStream in) throws IOException { - return DescriptorProto.parseDelimitedFrom(in, ExtensionRegistry.newInstance()); - } - }); + verifyExceptions( + new ParseTester() { + @Override + public DescriptorProto parse(InputStream in) throws IOException { + return DescriptorProto.parseDelimitedFrom(in, ExtensionRegistry.newInstance()); + } + }); } @Test public void messageBuilder_mergeFrom_InputStream() { setup(); - verifyExceptions(new ParseTester() { - public DescriptorProto parse(InputStream in) throws IOException { - return DescriptorProto.newBuilder().mergeFrom(in).build(); - } - }); + verifyExceptions( + new ParseTester() { + @Override + public DescriptorProto parse(InputStream in) throws IOException { + return DescriptorProto.newBuilder().mergeFrom(in).build(); + } + }); } @Test public void messageBuilder_mergeFrom_InputStreamAndExtensionRegistry() { setup(); - verifyExceptions(new ParseTester() { - public DescriptorProto parse(InputStream in) throws IOException { - return DescriptorProto.newBuilder().mergeFrom(in, ExtensionRegistry.newInstance()).build(); - } - }); + verifyExceptions( + new ParseTester() { + @Override + public DescriptorProto parse(InputStream in) throws IOException { + return DescriptorProto.newBuilder() + .mergeFrom(in, ExtensionRegistry.newInstance()) + .build(); + } + }); } @Test public void messageBuilder_mergeFrom_CodedInputStream() { setup(); - verifyExceptions(new ParseTester() { - public DescriptorProto parse(InputStream in) throws IOException { - return DescriptorProto.newBuilder().mergeFrom(CodedInputStream.newInstance(in)).build(); - } - }); + verifyExceptions( + new ParseTester() { + @Override + public DescriptorProto parse(InputStream in) throws IOException { + return DescriptorProto.newBuilder().mergeFrom(CodedInputStream.newInstance(in)).build(); + } + }); } @Test public void messageBuilder_mergeFrom_CodedInputStreamAndExtensionRegistry() { setup(); - verifyExceptions(new ParseTester() { - public DescriptorProto parse(InputStream in) throws IOException { - return DescriptorProto.newBuilder() - .mergeFrom(CodedInputStream.newInstance(in), ExtensionRegistry.newInstance()).build(); - } - }); + verifyExceptions( + new ParseTester() { + @Override + public DescriptorProto parse(InputStream in) throws IOException { + return DescriptorProto.newBuilder() + .mergeFrom(CodedInputStream.newInstance(in), ExtensionRegistry.newInstance()) + .build(); + } + }); } @Test public void messageBuilder_mergeDelimitedFrom_InputStream() { setupDelimited(); - verifyExceptions(new ParseTester() { - public DescriptorProto parse(InputStream in) throws IOException { - DescriptorProto.Builder builder = DescriptorProto.newBuilder(); - builder.mergeDelimitedFrom(in); - return builder.build(); - } - }); + verifyExceptions( + new ParseTester() { + @Override + public DescriptorProto parse(InputStream in) throws IOException { + DescriptorProto.Builder builder = DescriptorProto.newBuilder(); + builder.mergeDelimitedFrom(in); + return builder.build(); + } + }); } @Test public void messageBuilder_mergeDelimitedFrom_InputStreamAndExtensionRegistry() { setupDelimited(); - verifyExceptions(new ParseTester() { - public DescriptorProto parse(InputStream in) throws IOException { - DescriptorProto.Builder builder = DescriptorProto.newBuilder(); - builder.mergeDelimitedFrom(in, ExtensionRegistry.newInstance()); - return builder.build(); - } - }); + verifyExceptions( + new ParseTester() { + @Override + public DescriptorProto parse(InputStream in) throws IOException { + DescriptorProto.Builder builder = DescriptorProto.newBuilder(); + builder.mergeDelimitedFrom(in, ExtensionRegistry.newInstance()); + return builder.build(); + } + }); } private void verifyExceptions(ParseTester parseTester) { diff --git a/java/core/src/test/java/com/google/protobuf/ParserTest.java b/java/core/src/test/java/com/google/protobuf/ParserTest.java index 5a92bacf..9d4b6b94 100644 --- a/java/core/src/test/java/com/google/protobuf/ParserTest.java +++ b/java/core/src/test/java/com/google/protobuf/ParserTest.java @@ -33,15 +33,15 @@ package com.google.protobuf; import com.google.protobuf.UnittestLite.TestAllTypesLite; import com.google.protobuf.UnittestLite.TestPackedExtensionsLite; import com.google.protobuf.UnittestLite.TestParsingMergeLite; +import protobuf_unittest.UnittestOptimizeFor; import protobuf_unittest.UnittestOptimizeFor.TestOptimizedForSize; import protobuf_unittest.UnittestOptimizeFor.TestRequiredOptimizedForSize; -import protobuf_unittest.UnittestOptimizeFor; +import protobuf_unittest.UnittestProto; import protobuf_unittest.UnittestProto.ForeignMessage; import protobuf_unittest.UnittestProto.TestAllTypes; import protobuf_unittest.UnittestProto.TestEmptyMessage; import protobuf_unittest.UnittestProto.TestParsingMerge; import protobuf_unittest.UnittestProto.TestRequired; -import protobuf_unittest.UnittestProto; import junit.framework.TestCase; diff --git a/java/core/src/test/java/com/google/protobuf/ServiceTest.java b/java/core/src/test/java/com/google/protobuf/ServiceTest.java index ff980d66..7f3439d0 100644 --- a/java/core/src/test/java/com/google/protobuf/ServiceTest.java +++ b/java/core/src/test/java/com/google/protobuf/ServiceTest.java @@ -35,22 +35,22 @@ import com.google.protobuf.Descriptors.MethodDescriptor; import google.protobuf.no_generic_services_test.UnittestNoGenericServices; import protobuf_unittest.MessageWithNoOuter; import protobuf_unittest.ServiceWithNoOuter; -import protobuf_unittest.UnittestProto.TestAllTypes; -import protobuf_unittest.UnittestProto.TestService; -import protobuf_unittest.UnittestProto.FooRequest; -import protobuf_unittest.UnittestProto.FooResponse; import protobuf_unittest.UnittestProto.BarRequest; import protobuf_unittest.UnittestProto.BarResponse; +import protobuf_unittest.UnittestProto.FooRequest; +import protobuf_unittest.UnittestProto.FooResponse; +import protobuf_unittest.UnittestProto.TestAllTypes; +import protobuf_unittest.UnittestProto.TestService; + +import junit.framework.TestCase; import org.easymock.classextension.EasyMock; -import org.easymock.classextension.IMocksControl; import org.easymock.IArgumentMatcher; +import org.easymock.classextension.IMocksControl; import java.util.HashSet; import java.util.Set; -import junit.framework.TestCase; - /** * Tests services and stubs. * diff --git a/java/core/src/test/java/com/google/protobuf/TestUtil.java b/java/core/src/test/java/com/google/protobuf/TestUtil.java index 01acb884..53d65428 100644 --- a/java/core/src/test/java/com/google/protobuf/TestUtil.java +++ b/java/core/src/test/java/com/google/protobuf/TestUtil.java @@ -30,205 +30,207 @@ package com.google.protobuf; -import protobuf_unittest.UnittestProto; -import com.google.protobuf.UnittestLite; - +import static com.google.protobuf.UnittestLite.OptionalGroup_extension_lite; +import static com.google.protobuf.UnittestLite.RepeatedGroup_extension_lite; +import static com.google.protobuf.UnittestLite.defaultBoolExtensionLite; +import static com.google.protobuf.UnittestLite.defaultBytesExtensionLite; +import static com.google.protobuf.UnittestLite.defaultCordExtensionLite; +import static com.google.protobuf.UnittestLite.defaultDoubleExtensionLite; +import static com.google.protobuf.UnittestLite.defaultFixed32ExtensionLite; +import static com.google.protobuf.UnittestLite.defaultFixed64ExtensionLite; +import static com.google.protobuf.UnittestLite.defaultFloatExtensionLite; +import static com.google.protobuf.UnittestLite.defaultForeignEnumExtensionLite; +import static com.google.protobuf.UnittestLite.defaultImportEnumExtensionLite; +import static com.google.protobuf.UnittestLite.defaultInt32ExtensionLite; +import static com.google.protobuf.UnittestLite.defaultInt64ExtensionLite; +import static com.google.protobuf.UnittestLite.defaultNestedEnumExtensionLite; +import static com.google.protobuf.UnittestLite.defaultSfixed32ExtensionLite; +import static com.google.protobuf.UnittestLite.defaultSfixed64ExtensionLite; +import static com.google.protobuf.UnittestLite.defaultSint32ExtensionLite; +import static com.google.protobuf.UnittestLite.defaultSint64ExtensionLite; +import static com.google.protobuf.UnittestLite.defaultStringExtensionLite; +import static com.google.protobuf.UnittestLite.defaultStringPieceExtensionLite; +import static com.google.protobuf.UnittestLite.defaultUint32ExtensionLite; +import static com.google.protobuf.UnittestLite.defaultUint64ExtensionLite; +import static com.google.protobuf.UnittestLite.oneofBytesExtensionLite; +import static com.google.protobuf.UnittestLite.oneofNestedMessageExtensionLite; +import static com.google.protobuf.UnittestLite.oneofStringExtensionLite; +import static com.google.protobuf.UnittestLite.oneofUint32ExtensionLite; +import static com.google.protobuf.UnittestLite.optionalBoolExtensionLite; +import static com.google.protobuf.UnittestLite.optionalBytesExtensionLite; +import static com.google.protobuf.UnittestLite.optionalCordExtensionLite; +import static com.google.protobuf.UnittestLite.optionalDoubleExtensionLite; +import static com.google.protobuf.UnittestLite.optionalFixed32ExtensionLite; +import static com.google.protobuf.UnittestLite.optionalFixed64ExtensionLite; +import static com.google.protobuf.UnittestLite.optionalFloatExtensionLite; +import static com.google.protobuf.UnittestLite.optionalForeignEnumExtensionLite; +import static com.google.protobuf.UnittestLite.optionalForeignMessageExtensionLite; +import static com.google.protobuf.UnittestLite.optionalGroupExtensionLite; +import static com.google.protobuf.UnittestLite.optionalImportEnumExtensionLite; +import static com.google.protobuf.UnittestLite.optionalImportMessageExtensionLite; +import static com.google.protobuf.UnittestLite.optionalInt32ExtensionLite; +import static com.google.protobuf.UnittestLite.optionalInt64ExtensionLite; +import static com.google.protobuf.UnittestLite.optionalLazyMessageExtensionLite; +import static com.google.protobuf.UnittestLite.optionalNestedEnumExtensionLite; +import static com.google.protobuf.UnittestLite.optionalNestedMessageExtensionLite; +import static com.google.protobuf.UnittestLite.optionalPublicImportMessageExtensionLite; +import static com.google.protobuf.UnittestLite.optionalSfixed32ExtensionLite; +import static com.google.protobuf.UnittestLite.optionalSfixed64ExtensionLite; +import static com.google.protobuf.UnittestLite.optionalSint32ExtensionLite; +import static com.google.protobuf.UnittestLite.optionalSint64ExtensionLite; +import static com.google.protobuf.UnittestLite.optionalStringExtensionLite; +import static com.google.protobuf.UnittestLite.optionalStringPieceExtensionLite; +import static com.google.protobuf.UnittestLite.optionalUint32ExtensionLite; +import static com.google.protobuf.UnittestLite.optionalUint64ExtensionLite; +import static com.google.protobuf.UnittestLite.packedBoolExtensionLite; +import static com.google.protobuf.UnittestLite.packedDoubleExtensionLite; +import static com.google.protobuf.UnittestLite.packedEnumExtensionLite; +import static com.google.protobuf.UnittestLite.packedFixed32ExtensionLite; +import static com.google.protobuf.UnittestLite.packedFixed64ExtensionLite; +import static com.google.protobuf.UnittestLite.packedFloatExtensionLite; +import static com.google.protobuf.UnittestLite.packedInt32ExtensionLite; +import static com.google.protobuf.UnittestLite.packedInt64ExtensionLite; +import static com.google.protobuf.UnittestLite.packedSfixed32ExtensionLite; +import static com.google.protobuf.UnittestLite.packedSfixed64ExtensionLite; +import static com.google.protobuf.UnittestLite.packedSint32ExtensionLite; +import static com.google.protobuf.UnittestLite.packedSint64ExtensionLite; +import static com.google.protobuf.UnittestLite.packedUint32ExtensionLite; +import static com.google.protobuf.UnittestLite.packedUint64ExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedBoolExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedBytesExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedCordExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedDoubleExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedFixed32ExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedFixed64ExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedFloatExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedForeignEnumExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedForeignMessageExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedGroupExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedImportEnumExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedImportMessageExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedInt32ExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedInt64ExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedLazyMessageExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedNestedEnumExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedNestedMessageExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedSfixed32ExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedSfixed64ExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedSint32ExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedSint64ExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedStringExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedStringPieceExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedUint32ExtensionLite; +import static com.google.protobuf.UnittestLite.repeatedUint64ExtensionLite; +import static protobuf_unittest.UnittestProto.OptionalGroup_extension; +import static protobuf_unittest.UnittestProto.RepeatedGroup_extension; +import static protobuf_unittest.UnittestProto.defaultBoolExtension; +import static protobuf_unittest.UnittestProto.defaultBytesExtension; +import static protobuf_unittest.UnittestProto.defaultCordExtension; +import static protobuf_unittest.UnittestProto.defaultDoubleExtension; +import static protobuf_unittest.UnittestProto.defaultFixed32Extension; +import static protobuf_unittest.UnittestProto.defaultFixed64Extension; +import static protobuf_unittest.UnittestProto.defaultFloatExtension; +import static protobuf_unittest.UnittestProto.defaultForeignEnumExtension; +import static protobuf_unittest.UnittestProto.defaultImportEnumExtension; // The static imports are to avoid 100+ char lines. The following is roughly equivalent to // import static protobuf_unittest.UnittestProto.*; import static protobuf_unittest.UnittestProto.defaultInt32Extension; import static protobuf_unittest.UnittestProto.defaultInt64Extension; -import static protobuf_unittest.UnittestProto.defaultUint32Extension; -import static protobuf_unittest.UnittestProto.defaultUint64Extension; -import static protobuf_unittest.UnittestProto.defaultSint32Extension; -import static protobuf_unittest.UnittestProto.defaultSint64Extension; -import static protobuf_unittest.UnittestProto.defaultFixed32Extension; -import static protobuf_unittest.UnittestProto.defaultFixed64Extension; +import static protobuf_unittest.UnittestProto.defaultNestedEnumExtension; import static protobuf_unittest.UnittestProto.defaultSfixed32Extension; import static protobuf_unittest.UnittestProto.defaultSfixed64Extension; -import static protobuf_unittest.UnittestProto.defaultFloatExtension; -import static protobuf_unittest.UnittestProto.defaultDoubleExtension; -import static protobuf_unittest.UnittestProto.defaultBoolExtension; +import static protobuf_unittest.UnittestProto.defaultSint32Extension; +import static protobuf_unittest.UnittestProto.defaultSint64Extension; import static protobuf_unittest.UnittestProto.defaultStringExtension; -import static protobuf_unittest.UnittestProto.defaultBytesExtension; -import static protobuf_unittest.UnittestProto.defaultNestedEnumExtension; -import static protobuf_unittest.UnittestProto.defaultForeignEnumExtension; -import static protobuf_unittest.UnittestProto.defaultImportEnumExtension; import static protobuf_unittest.UnittestProto.defaultStringPieceExtension; -import static protobuf_unittest.UnittestProto.defaultCordExtension; - -import static protobuf_unittest.UnittestProto.oneofUint32Extension; +import static protobuf_unittest.UnittestProto.defaultUint32Extension; +import static protobuf_unittest.UnittestProto.defaultUint64Extension; +import static protobuf_unittest.UnittestProto.oneofBytesExtension; import static protobuf_unittest.UnittestProto.oneofNestedMessageExtension; import static protobuf_unittest.UnittestProto.oneofStringExtension; -import static protobuf_unittest.UnittestProto.oneofBytesExtension; - -import static protobuf_unittest.UnittestProto.optionalInt32Extension; -import static protobuf_unittest.UnittestProto.optionalInt64Extension; -import static protobuf_unittest.UnittestProto.optionalUint32Extension; -import static protobuf_unittest.UnittestProto.optionalUint64Extension; -import static protobuf_unittest.UnittestProto.optionalSint32Extension; -import static protobuf_unittest.UnittestProto.optionalSint64Extension; -import static protobuf_unittest.UnittestProto.optionalFixed32Extension; -import static protobuf_unittest.UnittestProto.optionalFixed64Extension; -import static protobuf_unittest.UnittestProto.optionalSfixed32Extension; -import static protobuf_unittest.UnittestProto.optionalSfixed64Extension; -import static protobuf_unittest.UnittestProto.optionalFloatExtension; -import static protobuf_unittest.UnittestProto.optionalDoubleExtension; +import static protobuf_unittest.UnittestProto.oneofUint32Extension; import static protobuf_unittest.UnittestProto.optionalBoolExtension; -import static protobuf_unittest.UnittestProto.optionalStringExtension; import static protobuf_unittest.UnittestProto.optionalBytesExtension; -import static protobuf_unittest.UnittestProto.optionalGroupExtension; import static protobuf_unittest.UnittestProto.optionalCordExtension; +import static protobuf_unittest.UnittestProto.optionalDoubleExtension; +import static protobuf_unittest.UnittestProto.optionalFixed32Extension; +import static protobuf_unittest.UnittestProto.optionalFixed64Extension; +import static protobuf_unittest.UnittestProto.optionalFloatExtension; import static protobuf_unittest.UnittestProto.optionalForeignEnumExtension; import static protobuf_unittest.UnittestProto.optionalForeignMessageExtension; +import static protobuf_unittest.UnittestProto.optionalGroupExtension; import static protobuf_unittest.UnittestProto.optionalImportEnumExtension; import static protobuf_unittest.UnittestProto.optionalImportMessageExtension; +import static protobuf_unittest.UnittestProto.optionalInt32Extension; +import static protobuf_unittest.UnittestProto.optionalInt64Extension; +import static protobuf_unittest.UnittestProto.optionalLazyMessageExtension; import static protobuf_unittest.UnittestProto.optionalNestedEnumExtension; import static protobuf_unittest.UnittestProto.optionalNestedMessageExtension; import static protobuf_unittest.UnittestProto.optionalPublicImportMessageExtension; -import static protobuf_unittest.UnittestProto.optionalLazyMessageExtension; +import static protobuf_unittest.UnittestProto.optionalSfixed32Extension; +import static protobuf_unittest.UnittestProto.optionalSfixed64Extension; +import static protobuf_unittest.UnittestProto.optionalSint32Extension; +import static protobuf_unittest.UnittestProto.optionalSint64Extension; +import static protobuf_unittest.UnittestProto.optionalStringExtension; import static protobuf_unittest.UnittestProto.optionalStringPieceExtension; - -import static protobuf_unittest.UnittestProto.repeatedInt32Extension; -import static protobuf_unittest.UnittestProto.repeatedInt64Extension; -import static protobuf_unittest.UnittestProto.repeatedUint32Extension; -import static protobuf_unittest.UnittestProto.repeatedUint64Extension; -import static protobuf_unittest.UnittestProto.repeatedSint32Extension; -import static protobuf_unittest.UnittestProto.repeatedSint64Extension; +import static protobuf_unittest.UnittestProto.optionalUint32Extension; +import static protobuf_unittest.UnittestProto.optionalUint64Extension; +import static protobuf_unittest.UnittestProto.packedBoolExtension; +import static protobuf_unittest.UnittestProto.packedDoubleExtension; +import static protobuf_unittest.UnittestProto.packedEnumExtension; +import static protobuf_unittest.UnittestProto.packedFixed32Extension; +import static protobuf_unittest.UnittestProto.packedFixed64Extension; +import static protobuf_unittest.UnittestProto.packedFloatExtension; +import static protobuf_unittest.UnittestProto.packedInt32Extension; +import static protobuf_unittest.UnittestProto.packedInt64Extension; +import static protobuf_unittest.UnittestProto.packedSfixed32Extension; +import static protobuf_unittest.UnittestProto.packedSfixed64Extension; +import static protobuf_unittest.UnittestProto.packedSint32Extension; +import static protobuf_unittest.UnittestProto.packedSint64Extension; +import static protobuf_unittest.UnittestProto.packedUint32Extension; +import static protobuf_unittest.UnittestProto.packedUint64Extension; +import static protobuf_unittest.UnittestProto.repeatedBoolExtension; +import static protobuf_unittest.UnittestProto.repeatedBytesExtension; +import static protobuf_unittest.UnittestProto.repeatedCordExtension; +import static protobuf_unittest.UnittestProto.repeatedDoubleExtension; import static protobuf_unittest.UnittestProto.repeatedFixed32Extension; import static protobuf_unittest.UnittestProto.repeatedFixed64Extension; -import static protobuf_unittest.UnittestProto.repeatedSfixed32Extension; -import static protobuf_unittest.UnittestProto.repeatedSfixed64Extension; import static protobuf_unittest.UnittestProto.repeatedFloatExtension; -import static protobuf_unittest.UnittestProto.repeatedDoubleExtension; -import static protobuf_unittest.UnittestProto.repeatedBoolExtension; -import static protobuf_unittest.UnittestProto.repeatedStringExtension; -import static protobuf_unittest.UnittestProto.repeatedBytesExtension; -import static protobuf_unittest.UnittestProto.repeatedGroupExtension; -import static protobuf_unittest.UnittestProto.repeatedNestedMessageExtension; +import static protobuf_unittest.UnittestProto.repeatedForeignEnumExtension; import static protobuf_unittest.UnittestProto.repeatedForeignMessageExtension; +import static protobuf_unittest.UnittestProto.repeatedGroupExtension; +import static protobuf_unittest.UnittestProto.repeatedImportEnumExtension; import static protobuf_unittest.UnittestProto.repeatedImportMessageExtension; +import static protobuf_unittest.UnittestProto.repeatedInt32Extension; +import static protobuf_unittest.UnittestProto.repeatedInt64Extension; import static protobuf_unittest.UnittestProto.repeatedLazyMessageExtension; import static protobuf_unittest.UnittestProto.repeatedNestedEnumExtension; -import static protobuf_unittest.UnittestProto.repeatedForeignEnumExtension; -import static protobuf_unittest.UnittestProto.repeatedImportEnumExtension; +import static protobuf_unittest.UnittestProto.repeatedNestedMessageExtension; +import static protobuf_unittest.UnittestProto.repeatedSfixed32Extension; +import static protobuf_unittest.UnittestProto.repeatedSfixed64Extension; +import static protobuf_unittest.UnittestProto.repeatedSint32Extension; +import static protobuf_unittest.UnittestProto.repeatedSint64Extension; +import static protobuf_unittest.UnittestProto.repeatedStringExtension; import static protobuf_unittest.UnittestProto.repeatedStringPieceExtension; -import static protobuf_unittest.UnittestProto.repeatedCordExtension; - -import static protobuf_unittest.UnittestProto.OptionalGroup_extension; -import static protobuf_unittest.UnittestProto.RepeatedGroup_extension; - -import static protobuf_unittest.UnittestProto.packedInt32Extension; -import static protobuf_unittest.UnittestProto.packedInt64Extension; -import static protobuf_unittest.UnittestProto.packedUint32Extension; -import static protobuf_unittest.UnittestProto.packedUint64Extension; -import static protobuf_unittest.UnittestProto.packedSint32Extension; -import static protobuf_unittest.UnittestProto.packedSint64Extension; -import static protobuf_unittest.UnittestProto.packedFixed32Extension; -import static protobuf_unittest.UnittestProto.packedFixed64Extension; -import static protobuf_unittest.UnittestProto.packedSfixed32Extension; -import static protobuf_unittest.UnittestProto.packedSfixed64Extension; -import static protobuf_unittest.UnittestProto.packedFloatExtension; -import static protobuf_unittest.UnittestProto.packedDoubleExtension; -import static protobuf_unittest.UnittestProto.packedBoolExtension; -import static protobuf_unittest.UnittestProto.packedEnumExtension; - -import static com.google.protobuf.UnittestLite.defaultInt32ExtensionLite; -import static com.google.protobuf.UnittestLite.defaultInt64ExtensionLite; -import static com.google.protobuf.UnittestLite.defaultUint32ExtensionLite; -import static com.google.protobuf.UnittestLite.defaultUint64ExtensionLite; -import static com.google.protobuf.UnittestLite.defaultSint32ExtensionLite; -import static com.google.protobuf.UnittestLite.defaultSint64ExtensionLite; -import static com.google.protobuf.UnittestLite.defaultFixed32ExtensionLite; -import static com.google.protobuf.UnittestLite.defaultFixed64ExtensionLite; -import static com.google.protobuf.UnittestLite.defaultSfixed32ExtensionLite; -import static com.google.protobuf.UnittestLite.defaultSfixed64ExtensionLite; -import static com.google.protobuf.UnittestLite.defaultFloatExtensionLite; -import static com.google.protobuf.UnittestLite.defaultDoubleExtensionLite; -import static com.google.protobuf.UnittestLite.defaultBoolExtensionLite; -import static com.google.protobuf.UnittestLite.defaultStringExtensionLite; -import static com.google.protobuf.UnittestLite.defaultBytesExtensionLite; -import static com.google.protobuf.UnittestLite.defaultNestedEnumExtensionLite; -import static com.google.protobuf.UnittestLite.defaultForeignEnumExtensionLite; -import static com.google.protobuf.UnittestLite.defaultImportEnumExtensionLite; -import static com.google.protobuf.UnittestLite.defaultStringPieceExtensionLite; -import static com.google.protobuf.UnittestLite.defaultCordExtensionLite; - -import static com.google.protobuf.UnittestLite.oneofUint32ExtensionLite; -import static com.google.protobuf.UnittestLite.oneofNestedMessageExtensionLite; -import static com.google.protobuf.UnittestLite.oneofStringExtensionLite; -import static com.google.protobuf.UnittestLite.oneofBytesExtensionLite; - -import static com.google.protobuf.UnittestLite.optionalInt32ExtensionLite; -import static com.google.protobuf.UnittestLite.optionalInt64ExtensionLite; -import static com.google.protobuf.UnittestLite.optionalUint32ExtensionLite; -import static com.google.protobuf.UnittestLite.optionalUint64ExtensionLite; -import static com.google.protobuf.UnittestLite.optionalSint32ExtensionLite; -import static com.google.protobuf.UnittestLite.optionalSint64ExtensionLite; -import static com.google.protobuf.UnittestLite.optionalFixed32ExtensionLite; -import static com.google.protobuf.UnittestLite.optionalFixed64ExtensionLite; -import static com.google.protobuf.UnittestLite.optionalSfixed32ExtensionLite; -import static com.google.protobuf.UnittestLite.optionalSfixed64ExtensionLite; -import static com.google.protobuf.UnittestLite.optionalFloatExtensionLite; -import static com.google.protobuf.UnittestLite.optionalDoubleExtensionLite; -import static com.google.protobuf.UnittestLite.optionalBoolExtensionLite; -import static com.google.protobuf.UnittestLite.optionalStringExtensionLite; -import static com.google.protobuf.UnittestLite.optionalBytesExtensionLite; -import static com.google.protobuf.UnittestLite.optionalGroupExtensionLite; -import static com.google.protobuf.UnittestLite.optionalNestedMessageExtensionLite; -import static com.google.protobuf.UnittestLite.optionalForeignEnumExtensionLite; -import static com.google.protobuf.UnittestLite.optionalForeignMessageExtensionLite; -import static com.google.protobuf.UnittestLite.optionalImportEnumExtensionLite; -import static com.google.protobuf.UnittestLite.optionalImportMessageExtensionLite; -import static com.google.protobuf.UnittestLite.optionalNestedEnumExtensionLite; -import static com.google.protobuf.UnittestLite.optionalPublicImportMessageExtensionLite; -import static com.google.protobuf.UnittestLite.optionalLazyMessageExtensionLite; -import static com.google.protobuf.UnittestLite.optionalStringPieceExtensionLite; -import static com.google.protobuf.UnittestLite.optionalCordExtensionLite; - -import static com.google.protobuf.UnittestLite.repeatedInt32ExtensionLite; -import static com.google.protobuf.UnittestLite.repeatedInt64ExtensionLite; -import static com.google.protobuf.UnittestLite.repeatedUint32ExtensionLite; -import static com.google.protobuf.UnittestLite.repeatedUint64ExtensionLite; -import static com.google.protobuf.UnittestLite.repeatedSint32ExtensionLite; -import static com.google.protobuf.UnittestLite.repeatedSint64ExtensionLite; -import static com.google.protobuf.UnittestLite.repeatedFixed32ExtensionLite; -import static com.google.protobuf.UnittestLite.repeatedFixed64ExtensionLite; -import static com.google.protobuf.UnittestLite.repeatedSfixed32ExtensionLite; -import static com.google.protobuf.UnittestLite.repeatedSfixed64ExtensionLite; -import static com.google.protobuf.UnittestLite.repeatedFloatExtensionLite; -import static com.google.protobuf.UnittestLite.repeatedDoubleExtensionLite; -import static com.google.protobuf.UnittestLite.repeatedBoolExtensionLite; -import static com.google.protobuf.UnittestLite.repeatedStringExtensionLite; -import static com.google.protobuf.UnittestLite.repeatedBytesExtensionLite; -import static com.google.protobuf.UnittestLite.repeatedGroupExtensionLite; -import static com.google.protobuf.UnittestLite.repeatedNestedMessageExtensionLite; -import static com.google.protobuf.UnittestLite.repeatedForeignMessageExtensionLite; -import static com.google.protobuf.UnittestLite.repeatedImportMessageExtensionLite; -import static com.google.protobuf.UnittestLite.repeatedLazyMessageExtensionLite; -import static com.google.protobuf.UnittestLite.repeatedNestedEnumExtensionLite; -import static com.google.protobuf.UnittestLite.repeatedForeignEnumExtensionLite; -import static com.google.protobuf.UnittestLite.repeatedImportEnumExtensionLite; -import static com.google.protobuf.UnittestLite.repeatedStringPieceExtensionLite; -import static com.google.protobuf.UnittestLite.repeatedCordExtensionLite; - -import static com.google.protobuf.UnittestLite.OptionalGroup_extension_lite; -import static com.google.protobuf.UnittestLite.RepeatedGroup_extension_lite; - -import static com.google.protobuf.UnittestLite.packedInt32ExtensionLite; -import static com.google.protobuf.UnittestLite.packedInt64ExtensionLite; -import static com.google.protobuf.UnittestLite.packedUint32ExtensionLite; -import static com.google.protobuf.UnittestLite.packedUint64ExtensionLite; -import static com.google.protobuf.UnittestLite.packedSint32ExtensionLite; -import static com.google.protobuf.UnittestLite.packedSint64ExtensionLite; -import static com.google.protobuf.UnittestLite.packedFixed32ExtensionLite; -import static com.google.protobuf.UnittestLite.packedFixed64ExtensionLite; -import static com.google.protobuf.UnittestLite.packedSfixed32ExtensionLite; -import static com.google.protobuf.UnittestLite.packedSfixed64ExtensionLite; -import static com.google.protobuf.UnittestLite.packedFloatExtensionLite; -import static com.google.protobuf.UnittestLite.packedDoubleExtensionLite; -import static com.google.protobuf.UnittestLite.packedBoolExtensionLite; -import static com.google.protobuf.UnittestLite.packedEnumExtensionLite; +import static protobuf_unittest.UnittestProto.repeatedUint32Extension; +import static protobuf_unittest.UnittestProto.repeatedUint64Extension; +import com.google.protobuf.UnittestImportLite.ImportEnumLite; +import com.google.protobuf.UnittestImportLite.ImportMessageLite; +import com.google.protobuf.UnittestImportPublicLite.PublicImportMessageLite; +import com.google.protobuf.UnittestLite; +import com.google.protobuf.UnittestLite.ForeignEnumLite; +import com.google.protobuf.UnittestLite.ForeignMessageLite; +import com.google.protobuf.UnittestLite.TestAllExtensionsLite; +import com.google.protobuf.UnittestLite.TestAllExtensionsLiteOrBuilder; +import com.google.protobuf.UnittestLite.TestAllTypesLite; +import com.google.protobuf.UnittestLite.TestPackedExtensionsLite; +import com.google.protobuf.test.UnittestImport.ImportEnum; +import com.google.protobuf.test.UnittestImport.ImportMessage; +import com.google.protobuf.test.UnittestImportPublic.PublicImportMessage; +import protobuf_unittest.UnittestProto; +import protobuf_unittest.UnittestProto.ForeignEnum; +import protobuf_unittest.UnittestProto.ForeignMessage; import protobuf_unittest.UnittestProto.TestAllExtensions; import protobuf_unittest.UnittestProto.TestAllExtensionsOrBuilder; import protobuf_unittest.UnittestProto.TestAllTypes; @@ -237,21 +239,6 @@ import protobuf_unittest.UnittestProto.TestOneof2; import protobuf_unittest.UnittestProto.TestPackedExtensions; import protobuf_unittest.UnittestProto.TestPackedTypes; import protobuf_unittest.UnittestProto.TestUnpackedTypes; -import protobuf_unittest.UnittestProto.ForeignMessage; -import protobuf_unittest.UnittestProto.ForeignEnum; -import com.google.protobuf.test.UnittestImport.ImportEnum; -import com.google.protobuf.test.UnittestImport.ImportMessage; -import com.google.protobuf.test.UnittestImportPublic.PublicImportMessage; - -import com.google.protobuf.UnittestLite.TestAllTypesLite; -import com.google.protobuf.UnittestLite.TestAllExtensionsLite; -import com.google.protobuf.UnittestLite.TestAllExtensionsLiteOrBuilder; -import com.google.protobuf.UnittestLite.TestPackedExtensionsLite; -import com.google.protobuf.UnittestLite.ForeignMessageLite; -import com.google.protobuf.UnittestLite.ForeignEnumLite; -import com.google.protobuf.UnittestImportLite.ImportEnumLite; -import com.google.protobuf.UnittestImportLite.ImportMessageLite; -import com.google.protobuf.UnittestImportPublicLite.PublicImportMessageLite; import junit.framework.Assert; diff --git a/java/core/src/test/java/com/google/protobuf/TextFormatParseInfoTreeTest.java b/java/core/src/test/java/com/google/protobuf/TextFormatParseInfoTreeTest.java new file mode 100644 index 00000000..e338af21 --- /dev/null +++ b/java/core/src/test/java/com/google/protobuf/TextFormatParseInfoTreeTest.java @@ -0,0 +1,182 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import com.google.protobuf.Descriptors.Descriptor; +import com.google.protobuf.Descriptors.FieldDescriptor; +import protobuf_unittest.UnittestProto.TestAllTypes; + +import junit.framework.TestCase; + +/** + * Test @{link TextFormatParseInfoTree}. + */ +public class TextFormatParseInfoTreeTest extends TestCase { + + private static final Descriptor DESCRIPTOR = TestAllTypes.getDescriptor(); + private static final FieldDescriptor OPTIONAL_INT32 = + DESCRIPTOR.findFieldByName("optional_int32"); + private static final FieldDescriptor OPTIONAL_BOOLEAN = + DESCRIPTOR.findFieldByName("optional_boolean"); + private static final FieldDescriptor REPEATED_INT32 = + DESCRIPTOR.findFieldByName("repeated_int32"); + private static final FieldDescriptor OPTIONAL_NESTED_MESSAGE = + DESCRIPTOR.findFieldByName("optional_nested_message"); + private static final FieldDescriptor REPEATED_NESTED_MESSAGE = + DESCRIPTOR.findFieldByName("repeated_nested_message"); + private static final FieldDescriptor FIELD_BB = + TestAllTypes.NestedMessage.getDescriptor().findFieldByName("bb"); + + private static final TextFormatParseLocation LOC0 = TextFormatParseLocation.create(1, 2); + private static final TextFormatParseLocation LOC1 = TextFormatParseLocation.create(2, 3); + + private TextFormatParseInfoTree.Builder rootBuilder; + + @Override + public void setUp() { + rootBuilder = TextFormatParseInfoTree.builder(); + } + + public void testBuildEmptyParseTree() { + TextFormatParseInfoTree tree = rootBuilder.build(); + assertTrue(tree.getLocations(null).isEmpty()); + } + + public void testGetLocationReturnsSingleLocation() { + rootBuilder.setLocation(OPTIONAL_INT32, LOC0); + TextFormatParseInfoTree root = rootBuilder.build(); + assertEquals(LOC0, root.getLocation(OPTIONAL_INT32, 0)); + assertEquals(1, root.getLocations(OPTIONAL_INT32).size()); + } + + public void testGetLocationsReturnsNoParseLocationsForUnknownField() { + assertTrue(rootBuilder.build().getLocations(OPTIONAL_INT32).isEmpty()); + rootBuilder.setLocation(OPTIONAL_BOOLEAN, LOC0); + TextFormatParseInfoTree root = rootBuilder.build(); + assertTrue(root.getLocations(OPTIONAL_INT32).isEmpty()); + assertEquals(LOC0, root.getLocations(OPTIONAL_BOOLEAN).get(0)); + } + + public void testGetLocationThrowsIllegalArgumentExceptionForUnknownField() { + rootBuilder.setLocation(REPEATED_INT32, LOC0); + TextFormatParseInfoTree root = rootBuilder.build(); + try { + root.getNestedTree(OPTIONAL_INT32, 0); + fail("Did not detect unknown field"); + } catch (IllegalArgumentException expected) { + // pass + } + } + + public void testGetLocationThrowsIllegalArgumentExceptionForInvalidIndex() { + TextFormatParseInfoTree root = rootBuilder.setLocation(OPTIONAL_INT32, LOC0).build(); + try { + root.getLocation(OPTIONAL_INT32, 1); + fail("Invalid index not detected"); + } catch (IllegalArgumentException expected) { + // pass + } + try { + root.getLocation(OPTIONAL_INT32, -1); + fail("Negative index not detected"); + } catch (IllegalArgumentException expected) { + // pass + } + } + + public void testGetLocationsReturnsMultipleLocations() { + rootBuilder.setLocation(REPEATED_INT32, LOC0); + rootBuilder.setLocation(REPEATED_INT32, LOC1); + TextFormatParseInfoTree root = rootBuilder.build(); + assertEquals(LOC0, root.getLocation(REPEATED_INT32, 0)); + assertEquals(LOC1, root.getLocation(REPEATED_INT32, 1)); + assertEquals(2, root.getLocations(REPEATED_INT32).size()); + } + + public void testGetNestedTreeThrowsIllegalArgumentExceptionForUnknownField() { + rootBuilder.setLocation(REPEATED_INT32, LOC0); + TextFormatParseInfoTree root = rootBuilder.build(); + try { + root.getNestedTree(OPTIONAL_NESTED_MESSAGE, 0); + fail("Did not detect unknown field"); + } catch (IllegalArgumentException expected) { + // pass + } + } + + public void testGetNestedTreesReturnsNoParseInfoTreesForUnknownField() { + rootBuilder.setLocation(REPEATED_INT32, LOC0); + TextFormatParseInfoTree root = rootBuilder.build(); + assertTrue(root.getNestedTrees(OPTIONAL_NESTED_MESSAGE).isEmpty()); + } + + public void testGetNestedTreeThrowsIllegalArgumentExceptionForInvalidIndex() { + rootBuilder.setLocation(REPEATED_INT32, LOC0); + rootBuilder.getBuilderForSubMessageField(OPTIONAL_NESTED_MESSAGE); + TextFormatParseInfoTree root = rootBuilder.build(); + try { + root.getNestedTree(OPTIONAL_NESTED_MESSAGE, 1); + fail("Submessage index that is too large not detected"); + } catch (IllegalArgumentException expected) { + // pass + } + try { + rootBuilder.build().getNestedTree(OPTIONAL_NESTED_MESSAGE, -1); + fail("Invalid submessage index (-1) not detected"); + } catch (IllegalArgumentException expected) { + // pass + } + } + + public void testGetNestedTreesReturnsSingleTree() { + rootBuilder.getBuilderForSubMessageField(OPTIONAL_NESTED_MESSAGE); + TextFormatParseInfoTree root = rootBuilder.build(); + assertEquals(1, root.getNestedTrees(OPTIONAL_NESTED_MESSAGE).size()); + TextFormatParseInfoTree subtree = root.getNestedTrees(OPTIONAL_NESTED_MESSAGE).get(0); + assertNotNull(subtree); + } + + public void testGetNestedTreesReturnsMultipleTrees() { + TextFormatParseInfoTree.Builder subtree1Builder = + rootBuilder.getBuilderForSubMessageField(REPEATED_NESTED_MESSAGE); + subtree1Builder.getBuilderForSubMessageField(FIELD_BB); + subtree1Builder.getBuilderForSubMessageField(FIELD_BB); + TextFormatParseInfoTree.Builder subtree2Builder = + rootBuilder.getBuilderForSubMessageField(REPEATED_NESTED_MESSAGE); + subtree2Builder.getBuilderForSubMessageField(FIELD_BB); + TextFormatParseInfoTree root = rootBuilder.build(); + assertEquals(2, root.getNestedTrees(REPEATED_NESTED_MESSAGE).size()); + assertEquals( + 2, root.getNestedTrees(REPEATED_NESTED_MESSAGE).get(0).getNestedTrees(FIELD_BB).size()); + assertEquals( + 1, root.getNestedTrees(REPEATED_NESTED_MESSAGE).get(1).getNestedTrees(FIELD_BB).size()); + } +} diff --git a/java/core/src/test/java/com/google/protobuf/TextFormatParseLocationTest.java b/java/core/src/test/java/com/google/protobuf/TextFormatParseLocationTest.java new file mode 100644 index 00000000..c42bfa6e --- /dev/null +++ b/java/core/src/test/java/com/google/protobuf/TextFormatParseLocationTest.java @@ -0,0 +1,86 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import junit.framework.TestCase; + +/** + * Test @{link TextFormatParseLocation}. + */ +public class TextFormatParseLocationTest extends TestCase { + + public void testCreateEmpty() { + TextFormatParseLocation location = TextFormatParseLocation.create(-1, -1); + assertEquals(TextFormatParseLocation.EMPTY, location); + } + + public void testCreate() { + TextFormatParseLocation location = TextFormatParseLocation.create(2, 1); + assertEquals(2, location.getLine()); + assertEquals(1, location.getColumn()); + } + + public void testCreateThrowsIllegalArgumentExceptionForInvalidIndex() { + try { + TextFormatParseLocation.create(-1, 0); + fail("Should throw IllegalArgumentException if line is less than 0"); + } catch (IllegalArgumentException unused) { + // pass + } + try { + TextFormatParseLocation.create(0, -1); + fail("Should throw, column < 0"); + } catch (IllegalArgumentException unused) { + // pass + } + } + + public void testHashCode() { + TextFormatParseLocation loc0 = TextFormatParseLocation.create(2, 1); + TextFormatParseLocation loc1 = TextFormatParseLocation.create(2, 1); + + assertEquals(loc0.hashCode(), loc1.hashCode()); + assertEquals( + TextFormatParseLocation.EMPTY.hashCode(), TextFormatParseLocation.EMPTY.hashCode()); + } + + public void testEquals() { + TextFormatParseLocation loc0 = TextFormatParseLocation.create(2, 1); + TextFormatParseLocation loc1 = TextFormatParseLocation.create(1, 2); + TextFormatParseLocation loc2 = TextFormatParseLocation.create(2, 2); + TextFormatParseLocation loc3 = TextFormatParseLocation.create(2, 1); + + assertEquals(loc0, loc3); + assertNotSame(loc0, loc1); + assertNotSame(loc0, loc2); + assertNotSame(loc1, loc2); + } +} diff --git a/java/core/src/test/java/com/google/protobuf/TextFormatTest.java b/java/core/src/test/java/com/google/protobuf/TextFormatTest.java index 1df4fad7..3f47d924 100644 --- a/java/core/src/test/java/com/google/protobuf/TextFormatTest.java +++ b/java/core/src/test/java/com/google/protobuf/TextFormatTest.java @@ -30,6 +30,7 @@ package com.google.protobuf; +import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.TextFormat.Parser.SingularOverwritePolicy; import protobuf_unittest.UnittestMset.TestMessageSetExtension1; @@ -45,6 +46,7 @@ import proto2_wireformat_unittest.UnittestMsetWireFormat.TestMessageSet; import junit.framework.TestCase; import java.io.StringReader; +import java.util.List; /** * Test case for {@link TextFormat}. @@ -568,6 +570,16 @@ public class TextFormatTest extends TestCase { assertEquals(kEscapeTestString, TextFormat.unescapeText(kEscapeTestStringEscaped)); + // Invariant + assertEquals("hello", + TextFormat.escapeBytes(bytes("hello"))); + assertEquals("hello", + TextFormat.escapeText("hello")); + assertEquals(bytes("hello"), + TextFormat.unescapeBytes("hello")); + assertEquals("hello", + TextFormat.unescapeText("hello")); + // Unicode handling. assertEquals("\\341\\210\\264", TextFormat.escapeText("\u1234")); assertEquals("\\341\\210\\264", @@ -811,7 +823,6 @@ public class TextFormatTest extends TestCase { private void assertPrintFieldValue(String expect, Object value, String fieldName) throws Exception { - TestAllTypes.Builder builder = TestAllTypes.newBuilder(); StringBuilder sb = new StringBuilder(); TextFormat.printFieldValue( TestAllTypes.getDescriptor().findFieldByName(fieldName), @@ -1018,4 +1029,98 @@ public class TextFormatTest extends TestCase { assertFalse(oneof.hasFooString()); assertTrue(oneof.hasFooInt()); } + + // ======================================================================= + // test location information + + public void testParseInfoTreeBuilding() throws Exception { + TestAllTypes.Builder builder = TestAllTypes.newBuilder(); + + Descriptor descriptor = TestAllTypes.getDescriptor(); + TextFormatParseInfoTree.Builder treeBuilder = TextFormatParseInfoTree.builder(); + // Set to allow unknown fields + TextFormat.Parser parser = + TextFormat.Parser.newBuilder() + .setParseInfoTreeBuilder(treeBuilder) + .build(); + + final String stringData = + "optional_int32: 1\n" + + "optional_int64: 2\n" + + " optional_double: 2.4\n" + + "repeated_int32: 5\n" + + "repeated_int32: 10\n" + + "optional_nested_message <\n" + + " bb: 78\n" + + ">\n" + + "repeated_nested_message <\n" + + " bb: 79\n" + + ">\n" + + "repeated_nested_message <\n" + + " bb: 80\n" + + ">"; + + parser.merge(stringData, builder); + TextFormatParseInfoTree tree = treeBuilder.build(); + + // Verify that the tree has the correct positions. + assertLocation(tree, descriptor, "optional_int32", 0, 0, 0); + assertLocation(tree, descriptor, "optional_int64", 0, 1, 0); + assertLocation(tree, descriptor, "optional_double", 0, 2, 2); + + assertLocation(tree, descriptor, "repeated_int32", 0, 3, 0); + assertLocation(tree, descriptor, "repeated_int32", 1, 4, 0); + + assertLocation(tree, descriptor, "optional_nested_message", 0, 5, 0); + assertLocation(tree, descriptor, "repeated_nested_message", 0, 8, 0); + assertLocation(tree, descriptor, "repeated_nested_message", 1, 11, 0); + + // Check for fields not set. For an invalid field, the location returned should be -1, -1. + assertLocation(tree, descriptor, "repeated_int64", 0, -1, -1); + assertLocation(tree, descriptor, "repeated_int32", 6, -1, -1); + + // Verify inside the nested message. + FieldDescriptor nestedField = descriptor.findFieldByName("optional_nested_message"); + + TextFormatParseInfoTree nestedTree = tree.getNestedTrees(nestedField).get(0); + assertLocation(nestedTree, nestedField.getMessageType(), "bb", 0, 6, 2); + + // Verify inside another nested message. + nestedField = descriptor.findFieldByName("repeated_nested_message"); + nestedTree = tree.getNestedTrees(nestedField).get(0); + assertLocation(nestedTree, nestedField.getMessageType(), "bb", 0, 9, 2); + + nestedTree = tree.getNestedTrees(nestedField).get(1); + assertLocation(nestedTree, nestedField.getMessageType(), "bb", 0, 12, 2); + + // Verify a NULL tree for an unknown nested field. + try { + tree.getNestedTree(nestedField, 2); + fail("unknown nested field should throw"); + } catch (IllegalArgumentException unused) { + // pass + } + } + + private void assertLocation( + TextFormatParseInfoTree tree, + final Descriptor descriptor, + final String fieldName, + int index, + int line, + int column) { + List<TextFormatParseLocation> locs = tree.getLocations(descriptor.findFieldByName(fieldName)); + if (index < locs.size()) { + TextFormatParseLocation location = locs.get(index); + TextFormatParseLocation expected = TextFormatParseLocation.create(line, column); + assertEquals(expected, location); + } else if (line != -1 && column != -1) { + fail( + String.format( + "Tree/descriptor/fieldname did not contain index %d, line %d column %d expected", + index, + line, + column)); + } + } } diff --git a/java/core/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java b/java/core/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java index dc987379..573cd665 100644 --- a/java/core/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java +++ b/java/core/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java @@ -47,46 +47,6 @@ import java.io.IOException; * @author dweis@google.com (Daniel Weis) */ public class UnknownFieldSetLiteTest extends TestCase { - - public void testNoDataIsDefaultInstance() { - assertSame( - UnknownFieldSetLite.getDefaultInstance(), - UnknownFieldSetLite.newBuilder() - .build()); - } - - public void testBuilderReuse() throws IOException { - UnknownFieldSetLite.Builder builder = UnknownFieldSetLite.newBuilder(); - builder.mergeVarintField(10, 2); - builder.build(); - - try { - builder.build(); - fail(); - } catch (UnsupportedOperationException e) { - // Expected. - } - - try { - builder.mergeFieldFrom(0, CodedInputStream.newInstance(new byte[0])); - fail(); - } catch (UnsupportedOperationException e) { - // Expected. - } - - try { - builder.mergeVarintField(5, 1); - fail(); - } catch (UnsupportedOperationException e) { - // Expected. - } - } - - public void testBuilderReuse_empty() { - UnknownFieldSetLite.Builder builder = UnknownFieldSetLite.newBuilder(); - builder.build(); - builder.build(); - } public void testDefaultInstance() { UnknownFieldSetLite unknownFields = UnknownFieldSetLite.getDefaultInstance(); diff --git a/java/core/src/test/java/com/google/protobuf/WireFormatTest.java b/java/core/src/test/java/com/google/protobuf/WireFormatTest.java index 0175005d..b3aabb8f 100644 --- a/java/core/src/test/java/com/google/protobuf/WireFormatTest.java +++ b/java/core/src/test/java/com/google/protobuf/WireFormatTest.java @@ -30,12 +30,11 @@ package com.google.protobuf; -import junit.framework.TestCase; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.util.List; - +import com.google.protobuf.UnittestLite.TestAllExtensionsLite; +import com.google.protobuf.UnittestLite.TestPackedExtensionsLite; +import protobuf_unittest.UnittestMset.RawMessageSet; +import protobuf_unittest.UnittestMset.TestMessageSetExtension1; +import protobuf_unittest.UnittestMset.TestMessageSetExtension2; import protobuf_unittest.UnittestProto; import protobuf_unittest.UnittestProto.TestAllExtensions; import protobuf_unittest.UnittestProto.TestAllTypes; @@ -44,12 +43,13 @@ import protobuf_unittest.UnittestProto.TestOneof2; import protobuf_unittest.UnittestProto.TestOneofBackwardsCompatible; import protobuf_unittest.UnittestProto.TestPackedExtensions; import protobuf_unittest.UnittestProto.TestPackedTypes; -import protobuf_unittest.UnittestMset.RawMessageSet; -import protobuf_unittest.UnittestMset.TestMessageSetExtension1; -import protobuf_unittest.UnittestMset.TestMessageSetExtension2; import proto2_wireformat_unittest.UnittestMsetWireFormat.TestMessageSet; -import com.google.protobuf.UnittestLite.TestAllExtensionsLite; -import com.google.protobuf.UnittestLite.TestPackedExtensionsLite; + +import junit.framework.TestCase; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.util.List; /** * Tests related to parsing and serialization. diff --git a/java/core/src/test/proto/com/google/protobuf/lite_equals_and_hash.proto b/java/core/src/test/proto/com/google/protobuf/lite_equals_and_hash.proto index 86837250..f75484de 100644 --- a/java/core/src/test/proto/com/google/protobuf/lite_equals_and_hash.proto +++ b/java/core/src/test/proto/com/google/protobuf/lite_equals_and_hash.proto @@ -39,6 +39,13 @@ package protobuf_unittest.lite_equals_and_hash; option java_generate_equals_and_hash = true; option optimize_for = LITE_RUNTIME; +message TestOneofEquals { + oneof oneof_field { + string name = 1; + int32 value = 2; + } +} + message Foo { optional int32 value = 1; repeated Bar bar = 2; @@ -70,3 +77,8 @@ extend Foo { } } +message TestRecursiveOneof { + oneof Foo { + TestRecursiveOneof r = 1; + } +} diff --git a/java/pom.xml b/java/pom.xml index d5719edf..787bc534 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -64,7 +64,7 @@ <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> - <version>4.4</version> + <version>4.12</version> <scope>test</scope> </dependency> <dependency> diff --git a/java/util/src/main/java/com/google/protobuf/util/FieldMaskTree.java b/java/util/src/main/java/com/google/protobuf/util/FieldMaskTree.java index dc2f4b84..d6a2f6fa 100644 --- a/java/util/src/main/java/com/google/protobuf/util/FieldMaskTree.java +++ b/java/util/src/main/java/com/google/protobuf/util/FieldMaskTree.java @@ -60,10 +60,9 @@ import java.util.logging.Logger; * FieldMask in a message tree. */ class FieldMaskTree { - private static final Logger logger = - Logger.getLogger(FieldMaskTree.class.getName()); - - private static final String FIELD_PATH_SEPARATOR_REGEX = "\\."; + private static final Logger logger = Logger.getLogger(FieldMaskTree.class.getName()); + + private static final String FIELD_PATH_SEPARATOR_REGEX = "\\."; private static class Node { public TreeMap<String, Node> children = new TreeMap<String, Node>(); @@ -73,12 +72,12 @@ class FieldMaskTree { /** Creates an empty FieldMaskTree. */ public FieldMaskTree() {} - + /** Creates a FieldMaskTree for a given FieldMask. */ public FieldMaskTree(FieldMask mask) { mergeFromFieldMask(mask); } - + @Override public String toString() { return FieldMaskUtil.toString(toFieldMask()); @@ -121,7 +120,7 @@ class FieldMaskTree { node.children.clear(); return this; } - + /** * Merges all field paths in a FieldMask into this tree. */ @@ -149,8 +148,7 @@ class FieldMaskTree { return; } for (Entry<String, Node> entry : node.children.entrySet()) { - String childPath = path.isEmpty() - ? entry.getKey() : path + "." + entry.getKey(); + String childPath = path.isEmpty() ? entry.getKey() : path + "." + entry.getKey(); getFieldPaths(entry.getValue(), childPath, paths); } } @@ -193,11 +191,10 @@ class FieldMaskTree { * Merges all fields specified by this FieldMaskTree from {@code source} to * {@code destination}. */ - public void merge(Message source, Message.Builder destination, - FieldMaskUtil.MergeOptions options) { + public void merge( + Message source, Message.Builder destination, FieldMaskUtil.MergeOptions options) { if (source.getDescriptorForType() != destination.getDescriptorForType()) { - throw new IllegalArgumentException( - "Cannot merge messages of different types."); + throw new IllegalArgumentException("Cannot merge messages of different types."); } if (root.children.isEmpty()) { return; @@ -208,30 +205,41 @@ class FieldMaskTree { /** Merges all fields specified by a sub-tree from {@code source} to * {@code destination}. */ - private void merge(Node node, String path, Message source, - Message.Builder destination, FieldMaskUtil.MergeOptions options) { + private void merge( + Node node, + String path, + Message source, + Message.Builder destination, + FieldMaskUtil.MergeOptions options) { assert source.getDescriptorForType() == destination.getDescriptorForType(); - + Descriptor descriptor = source.getDescriptorForType(); for (Entry<String, Node> entry : node.children.entrySet()) { - FieldDescriptor field = - descriptor.findFieldByName(entry.getKey()); + FieldDescriptor field = descriptor.findFieldByName(entry.getKey()); if (field == null) { - logger.warning("Cannot find field \"" + entry.getKey() - + "\" in message type " + descriptor.getFullName()); + logger.warning( + "Cannot find field \"" + + entry.getKey() + + "\" in message type " + + descriptor.getFullName()); continue; } if (!entry.getValue().children.isEmpty()) { - if (field.isRepeated() - || field.getJavaType() != FieldDescriptor.JavaType.MESSAGE) { - logger.warning("Field \"" + field.getFullName() + "\" is not a " - + "singluar message field and cannot have sub-fields."); + if (field.isRepeated() || field.getJavaType() != FieldDescriptor.JavaType.MESSAGE) { + logger.warning( + "Field \"" + + field.getFullName() + + "\" is not a " + + "singluar message field and cannot have sub-fields."); continue; } - String childPath = path.isEmpty() - ? entry.getKey() : path + "." + entry.getKey(); - merge(entry.getValue(), childPath, (Message) source.getField(field), - destination.getFieldBuilder(field), options); + String childPath = path.isEmpty() ? entry.getKey() : path + "." + entry.getKey(); + merge( + entry.getValue(), + childPath, + (Message) source.getField(field), + destination.getFieldBuilder(field), + options); continue; } if (field.isRepeated()) { @@ -245,10 +253,15 @@ class FieldMaskTree { } else { if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { if (options.replaceMessageFields()) { - destination.setField(field, source.getField(field)); + if (!source.hasField(field)) { + destination.clearField(field); + } else { + destination.setField(field, source.getField(field)); + } } else { - destination.getFieldBuilder(field).mergeFrom( - (Message) source.getField(field)); + if (source.hasField(field)) { + destination.getFieldBuilder(field).mergeFrom((Message) source.getField(field)); + } } } else { destination.setField(field, source.getField(field)); diff --git a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java index d13ff0ed..d7bf34a4 100644 --- a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java +++ b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java @@ -32,6 +32,7 @@ package com.google.protobuf.util; import com.google.common.io.BaseEncoding; import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonNull; @@ -433,7 +434,7 @@ public class JsonFormat { private final Gson gson; private static class GsonHolder { - private static final Gson DEFAULT_GSON = new Gson(); + private static final Gson DEFAULT_GSON = new GsonBuilder().disableHtmlEscaping().create(); } PrinterImpl( @@ -1066,6 +1067,15 @@ public class JsonFormat { parser.mergeStruct(json, builder); } }); + // Special-case ListValue. + parsers.put(ListValue.getDescriptor().getFullName(), + new WellKnownTypeParser() { + @Override + public void merge(ParserImpl parser, JsonElement json, + Message.Builder builder) throws InvalidProtocolBufferException { + parser.mergeListValue(json, builder); + } + }); // Special-case Value. parsers.put(Value.getDescriptor().getFullName(), new WellKnownTypeParser() { @@ -1213,6 +1223,16 @@ public class JsonFormat { } mergeMapField(field, json, builder); } + + private void mergeListValue(JsonElement json, Message.Builder builder) + throws InvalidProtocolBufferException { + Descriptor descriptor = builder.getDescriptorForType(); + FieldDescriptor field = descriptor.findFieldByName("values"); + if (field == null) { + throw new InvalidProtocolBufferException("Invalid ListValue type."); + } + mergeRepeatedField(field, json, builder); + } private void mergeValue(JsonElement json, Message.Builder builder) throws InvalidProtocolBufferException { @@ -1237,9 +1257,7 @@ public class JsonFormat { } else if (json instanceof JsonArray) { FieldDescriptor field = type.findFieldByName("list_value"); Message.Builder listBuilder = builder.newBuilderForField(field); - FieldDescriptor listField = - listBuilder.getDescriptorForType().findFieldByName("values"); - mergeRepeatedField(listField, json, listBuilder); + merge(json, listBuilder); builder.setField(field, listBuilder.build()); } else { throw new IllegalStateException("Unexpected json data: " + json); diff --git a/java/util/src/test/java/com/google/protobuf/util/FieldMaskTreeTest.java b/java/util/src/test/java/com/google/protobuf/util/FieldMaskTreeTest.java index 3391f239..618e0072 100644 --- a/java/util/src/test/java/com/google/protobuf/util/FieldMaskTreeTest.java +++ b/java/util/src/test/java/com/google/protobuf/util/FieldMaskTreeTest.java @@ -61,19 +61,16 @@ public class FieldMaskTreeTest extends TestCase { tree.addFieldPath("bar"); assertEquals("bar,foo", tree.toString()); } - + public void testMergeFromFieldMask() throws Exception { - FieldMaskTree tree = new FieldMaskTree( - FieldMaskUtil.fromString("foo,bar.baz,bar.quz")); + FieldMaskTree tree = new FieldMaskTree(FieldMaskUtil.fromString("foo,bar.baz,bar.quz")); assertEquals("bar.baz,bar.quz,foo", tree.toString()); - tree.mergeFromFieldMask( - FieldMaskUtil.fromString("foo.bar,bar")); + tree.mergeFromFieldMask(FieldMaskUtil.fromString("foo.bar,bar")); assertEquals("bar,foo", tree.toString()); } - + public void testIntersectFieldPath() throws Exception { - FieldMaskTree tree = new FieldMaskTree( - FieldMaskUtil.fromString("foo,bar.baz,bar.quz")); + FieldMaskTree tree = new FieldMaskTree(FieldMaskUtil.fromString("foo,bar.baz,bar.quz")); FieldMaskTree result = new FieldMaskTree(); // Empty path. tree.intersectFieldPath("", result); @@ -96,16 +93,18 @@ public class FieldMaskTreeTest extends TestCase { } public void testMerge() throws Exception { - TestAllTypes value = TestAllTypes.newBuilder() - .setOptionalInt32(1234) - .setOptionalNestedMessage(NestedMessage.newBuilder().setBb(5678)) - .addRepeatedInt32(4321) - .addRepeatedNestedMessage(NestedMessage.newBuilder().setBb(8765)) - .build(); - NestedTestAllTypes source = NestedTestAllTypes.newBuilder() - .setPayload(value) - .setChild(NestedTestAllTypes.newBuilder().setPayload(value)) - .build(); + TestAllTypes value = + TestAllTypes.newBuilder() + .setOptionalInt32(1234) + .setOptionalNestedMessage(NestedMessage.newBuilder().setBb(5678)) + .addRepeatedInt32(4321) + .addRepeatedNestedMessage(NestedMessage.newBuilder().setBb(8765)) + .build(); + NestedTestAllTypes source = + NestedTestAllTypes.newBuilder() + .setPayload(value) + .setChild(NestedTestAllTypes.newBuilder().setPayload(value)) + .build(); // Now we have a message source with the following structure: // [root] -+- payload -+- optional_int32 // | +- optional_nested_message @@ -116,114 +115,127 @@ public class FieldMaskTreeTest extends TestCase { // +- optional_nested_message // +- repeated_int32 // +- repeated_nested_message - + FieldMaskUtil.MergeOptions options = new FieldMaskUtil.MergeOptions(); - + // Test merging each individual field. NestedTestAllTypes.Builder builder = NestedTestAllTypes.newBuilder(); - new FieldMaskTree().addFieldPath("payload.optional_int32") - .merge(source, builder, options); + new FieldMaskTree().addFieldPath("payload.optional_int32").merge(source, builder, options); NestedTestAllTypes.Builder expected = NestedTestAllTypes.newBuilder(); expected.getPayloadBuilder().setOptionalInt32(1234); assertEquals(expected.build(), builder.build()); builder = NestedTestAllTypes.newBuilder(); - new FieldMaskTree().addFieldPath("payload.optional_nested_message") + new FieldMaskTree() + .addFieldPath("payload.optional_nested_message") .merge(source, builder, options); expected = NestedTestAllTypes.newBuilder(); - expected.getPayloadBuilder().setOptionalNestedMessage( - NestedMessage.newBuilder().setBb(5678)); + expected.getPayloadBuilder().setOptionalNestedMessage(NestedMessage.newBuilder().setBb(5678)); assertEquals(expected.build(), builder.build()); - builder = NestedTestAllTypes.newBuilder(); - new FieldMaskTree().addFieldPath("payload.repeated_int32") - .merge(source, builder, options); + new FieldMaskTree().addFieldPath("payload.repeated_int32").merge(source, builder, options); expected = NestedTestAllTypes.newBuilder(); expected.getPayloadBuilder().addRepeatedInt32(4321); assertEquals(expected.build(), builder.build()); builder = NestedTestAllTypes.newBuilder(); - new FieldMaskTree().addFieldPath("payload.repeated_nested_message") + new FieldMaskTree() + .addFieldPath("payload.repeated_nested_message") .merge(source, builder, options); expected = NestedTestAllTypes.newBuilder(); - expected.getPayloadBuilder().addRepeatedNestedMessage( - NestedMessage.newBuilder().setBb(8765)); + expected.getPayloadBuilder().addRepeatedNestedMessage(NestedMessage.newBuilder().setBb(8765)); assertEquals(expected.build(), builder.build()); builder = NestedTestAllTypes.newBuilder(); - new FieldMaskTree().addFieldPath("child.payload.optional_int32") + new FieldMaskTree() + .addFieldPath("child.payload.optional_int32") .merge(source, builder, options); expected = NestedTestAllTypes.newBuilder(); expected.getChildBuilder().getPayloadBuilder().setOptionalInt32(1234); assertEquals(expected.build(), builder.build()); builder = NestedTestAllTypes.newBuilder(); - new FieldMaskTree().addFieldPath("child.payload.optional_nested_message") + new FieldMaskTree() + .addFieldPath("child.payload.optional_nested_message") .merge(source, builder, options); expected = NestedTestAllTypes.newBuilder(); - expected.getChildBuilder().getPayloadBuilder().setOptionalNestedMessage( - NestedMessage.newBuilder().setBb(5678)); + expected + .getChildBuilder() + .getPayloadBuilder() + .setOptionalNestedMessage(NestedMessage.newBuilder().setBb(5678)); assertEquals(expected.build(), builder.build()); - builder = NestedTestAllTypes.newBuilder(); - new FieldMaskTree().addFieldPath("child.payload.repeated_int32") + new FieldMaskTree() + .addFieldPath("child.payload.repeated_int32") .merge(source, builder, options); expected = NestedTestAllTypes.newBuilder(); expected.getChildBuilder().getPayloadBuilder().addRepeatedInt32(4321); assertEquals(expected.build(), builder.build()); - builder = NestedTestAllTypes.newBuilder(); - new FieldMaskTree().addFieldPath("child.payload.repeated_nested_message") + new FieldMaskTree() + .addFieldPath("child.payload.repeated_nested_message") .merge(source, builder, options); expected = NestedTestAllTypes.newBuilder(); - expected.getChildBuilder().getPayloadBuilder().addRepeatedNestedMessage( - NestedMessage.newBuilder().setBb(8765)); + expected + .getChildBuilder() + .getPayloadBuilder() + .addRepeatedNestedMessage(NestedMessage.newBuilder().setBb(8765)); assertEquals(expected.build(), builder.build()); - + // Test merging all fields. builder = NestedTestAllTypes.newBuilder(); - new FieldMaskTree().addFieldPath("child").addFieldPath("payload") - .merge(source, builder, options); + new FieldMaskTree() + .addFieldPath("child") + .addFieldPath("payload") + .merge(source, builder, options); assertEquals(source, builder.build()); - + // Test repeated options. builder = NestedTestAllTypes.newBuilder(); builder.getPayloadBuilder().addRepeatedInt32(1000); - new FieldMaskTree().addFieldPath("payload.repeated_int32") - .merge(source, builder, options); + new FieldMaskTree().addFieldPath("payload.repeated_int32").merge(source, builder, options); // Default behavior is to append repeated fields. assertEquals(2, builder.getPayload().getRepeatedInt32Count()); assertEquals(1000, builder.getPayload().getRepeatedInt32(0)); assertEquals(4321, builder.getPayload().getRepeatedInt32(1)); // Change to replace repeated fields. options.setReplaceRepeatedFields(true); - new FieldMaskTree().addFieldPath("payload.repeated_int32") - .merge(source, builder, options); + new FieldMaskTree().addFieldPath("payload.repeated_int32").merge(source, builder, options); assertEquals(1, builder.getPayload().getRepeatedInt32Count()); assertEquals(4321, builder.getPayload().getRepeatedInt32(0)); - + // Test message options. builder = NestedTestAllTypes.newBuilder(); builder.getPayloadBuilder().setOptionalInt32(1000); builder.getPayloadBuilder().setOptionalUint32(2000); - new FieldMaskTree().addFieldPath("payload") - .merge(source, builder, options); + new FieldMaskTree().addFieldPath("payload").merge(source, builder, options); // Default behavior is to merge message fields. assertEquals(1234, builder.getPayload().getOptionalInt32()); assertEquals(2000, builder.getPayload().getOptionalUint32()); - + + // Test merging unset message fields. + NestedTestAllTypes clearedSource = source.toBuilder().clearPayload().build(); + builder = NestedTestAllTypes.newBuilder(); + new FieldMaskTree().addFieldPath("payload").merge(clearedSource, builder, options); + assertEquals(false, builder.hasPayload()); + // Change to replace message fields. options.setReplaceMessageFields(true); builder = NestedTestAllTypes.newBuilder(); builder.getPayloadBuilder().setOptionalInt32(1000); builder.getPayloadBuilder().setOptionalUint32(2000); - new FieldMaskTree().addFieldPath("payload") - .merge(source, builder, options); + new FieldMaskTree().addFieldPath("payload").merge(source, builder, options); assertEquals(1234, builder.getPayload().getOptionalInt32()); assertEquals(0, builder.getPayload().getOptionalUint32()); + + // Test merging unset message fields. + builder = NestedTestAllTypes.newBuilder(); + builder.getPayloadBuilder().setOptionalInt32(1000); + builder.getPayloadBuilder().setOptionalUint32(2000); + new FieldMaskTree().addFieldPath("payload").merge(clearedSource, builder, options); + assertEquals(false, builder.hasPayload()); } } - diff --git a/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java b/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java index c0eb0330..d95b626c 100644 --- a/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java +++ b/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java @@ -122,11 +122,11 @@ public class JsonFormatTest extends TestCase { builder.addRepeatedNestedEnum(NestedEnum.BAZ); builder.addRepeatedNestedMessageBuilder().setValue(200); } - + private void assertRoundTripEquals(Message message) throws Exception { assertRoundTripEquals(message, TypeRegistry.getEmptyTypeRegistry()); } - + private void assertRoundTripEquals(Message message, TypeRegistry registry) throws Exception { JsonFormat.Printer printer = JsonFormat.printer().usingTypeRegistry(registry); JsonFormat.Parser parser = JsonFormat.parser().usingTypeRegistry(registry); @@ -135,68 +135,68 @@ public class JsonFormatTest extends TestCase { Message parsedMessage = builder.build(); assertEquals(message.toString(), parsedMessage.toString()); } - + private String toJsonString(Message message) throws IOException { return JsonFormat.printer().print(message); } - + private void mergeFromJson(String json, Message.Builder builder) throws IOException { JsonFormat.parser().merge(json, builder); } - + public void testAllFields() throws Exception { TestAllTypes.Builder builder = TestAllTypes.newBuilder(); setAllFields(builder); TestAllTypes message = builder.build(); - - assertEquals( + + assertEquals( "{\n" - + " \"optionalInt32\": 1234,\n" - + " \"optionalInt64\": \"1234567890123456789\",\n" - + " \"optionalUint32\": 5678,\n" - + " \"optionalUint64\": \"2345678901234567890\",\n" - + " \"optionalSint32\": 9012,\n" - + " \"optionalSint64\": \"3456789012345678901\",\n" - + " \"optionalFixed32\": 3456,\n" - + " \"optionalFixed64\": \"4567890123456789012\",\n" - + " \"optionalSfixed32\": 7890,\n" - + " \"optionalSfixed64\": \"5678901234567890123\",\n" - + " \"optionalFloat\": 1.5,\n" - + " \"optionalDouble\": 1.25,\n" - + " \"optionalBool\": true,\n" - + " \"optionalString\": \"Hello world!\",\n" - + " \"optionalBytes\": \"AAEC\",\n" - + " \"optionalNestedMessage\": {\n" - + " \"value\": 100\n" - + " },\n" - + " \"optionalNestedEnum\": \"BAR\",\n" - + " \"repeatedInt32\": [1234, 234],\n" - + " \"repeatedInt64\": [\"1234567890123456789\", \"234567890123456789\"],\n" - + " \"repeatedUint32\": [5678, 678],\n" - + " \"repeatedUint64\": [\"2345678901234567890\", \"345678901234567890\"],\n" - + " \"repeatedSint32\": [9012, 10],\n" - + " \"repeatedSint64\": [\"3456789012345678901\", \"456789012345678901\"],\n" - + " \"repeatedFixed32\": [3456, 456],\n" - + " \"repeatedFixed64\": [\"4567890123456789012\", \"567890123456789012\"],\n" - + " \"repeatedSfixed32\": [7890, 890],\n" - + " \"repeatedSfixed64\": [\"5678901234567890123\", \"678901234567890123\"],\n" - + " \"repeatedFloat\": [1.5, 11.5],\n" - + " \"repeatedDouble\": [1.25, 11.25],\n" - + " \"repeatedBool\": [true, true],\n" - + " \"repeatedString\": [\"Hello world!\", \"ello world!\"],\n" - + " \"repeatedBytes\": [\"AAEC\", \"AQI=\"],\n" - + " \"repeatedNestedMessage\": [{\n" - + " \"value\": 100\n" - + " }, {\n" - + " \"value\": 200\n" - + " }],\n" - + " \"repeatedNestedEnum\": [\"BAR\", \"BAZ\"]\n" - + "}", + + " \"optionalInt32\": 1234,\n" + + " \"optionalInt64\": \"1234567890123456789\",\n" + + " \"optionalUint32\": 5678,\n" + + " \"optionalUint64\": \"2345678901234567890\",\n" + + " \"optionalSint32\": 9012,\n" + + " \"optionalSint64\": \"3456789012345678901\",\n" + + " \"optionalFixed32\": 3456,\n" + + " \"optionalFixed64\": \"4567890123456789012\",\n" + + " \"optionalSfixed32\": 7890,\n" + + " \"optionalSfixed64\": \"5678901234567890123\",\n" + + " \"optionalFloat\": 1.5,\n" + + " \"optionalDouble\": 1.25,\n" + + " \"optionalBool\": true,\n" + + " \"optionalString\": \"Hello world!\",\n" + + " \"optionalBytes\": \"AAEC\",\n" + + " \"optionalNestedMessage\": {\n" + + " \"value\": 100\n" + + " },\n" + + " \"optionalNestedEnum\": \"BAR\",\n" + + " \"repeatedInt32\": [1234, 234],\n" + + " \"repeatedInt64\": [\"1234567890123456789\", \"234567890123456789\"],\n" + + " \"repeatedUint32\": [5678, 678],\n" + + " \"repeatedUint64\": [\"2345678901234567890\", \"345678901234567890\"],\n" + + " \"repeatedSint32\": [9012, 10],\n" + + " \"repeatedSint64\": [\"3456789012345678901\", \"456789012345678901\"],\n" + + " \"repeatedFixed32\": [3456, 456],\n" + + " \"repeatedFixed64\": [\"4567890123456789012\", \"567890123456789012\"],\n" + + " \"repeatedSfixed32\": [7890, 890],\n" + + " \"repeatedSfixed64\": [\"5678901234567890123\", \"678901234567890123\"],\n" + + " \"repeatedFloat\": [1.5, 11.5],\n" + + " \"repeatedDouble\": [1.25, 11.25],\n" + + " \"repeatedBool\": [true, true],\n" + + " \"repeatedString\": [\"Hello world!\", \"ello world!\"],\n" + + " \"repeatedBytes\": [\"AAEC\", \"AQI=\"],\n" + + " \"repeatedNestedMessage\": [{\n" + + " \"value\": 100\n" + + " }, {\n" + + " \"value\": 200\n" + + " }],\n" + + " \"repeatedNestedEnum\": [\"BAR\", \"BAZ\"]\n" + + "}", toJsonString(message)); - + assertRoundTripEquals(message); } - + public void testUnknownEnumValues() throws Exception { TestAllTypes message = TestAllTypes.newBuilder() .setOptionalNestedEnumValue(12345) @@ -209,21 +209,22 @@ public class JsonFormatTest extends TestCase { + " \"repeatedNestedEnum\": [12345, \"FOO\"]\n" + "}", toJsonString(message)); assertRoundTripEquals(message); - + TestMap.Builder mapBuilder = TestMap.newBuilder(); mapBuilder.getMutableInt32ToEnumMapValue().put(1, 0); mapBuilder.getMutableInt32ToEnumMapValue().put(2, 12345); TestMap mapMessage = mapBuilder.build(); assertEquals( - "{\n" - + " \"int32ToEnumMap\": {\n" - + " \"1\": \"FOO\",\n" - + " \"2\": 12345\n" - + " }\n" - + "}", toJsonString(mapMessage)); + "{\n" + + " \"int32ToEnumMap\": {\n" + + " \"1\": \"FOO\",\n" + + " \"2\": 12345\n" + + " }\n" + + "}", + toJsonString(mapMessage)); assertRoundTripEquals(mapMessage); } - + public void testSpecialFloatValues() throws Exception { TestAllTypes message = TestAllTypes.newBuilder() .addRepeatedFloat(Float.NaN) @@ -238,10 +239,10 @@ public class JsonFormatTest extends TestCase { + " \"repeatedFloat\": [\"NaN\", \"Infinity\", \"-Infinity\"],\n" + " \"repeatedDouble\": [\"NaN\", \"Infinity\", \"-Infinity\"]\n" + "}", toJsonString(message)); - + assertRoundTripEquals(message); } - + public void testParserAcceptStringForNumbericField() throws Exception { TestAllTypes.Builder builder = TestAllTypes.newBuilder(); mergeFromJson( @@ -265,7 +266,7 @@ public class JsonFormatTest extends TestCase { assertEquals(1.25, message.getOptionalDouble()); assertEquals(true, message.getOptionalBool()); } - + public void testParserAcceptFloatingPointValueForIntegerField() throws Exception { // Test that numeric values like "1.000", "1e5" will also be accepted. TestAllTypes.Builder builder = TestAllTypes.newBuilder(); @@ -287,14 +288,14 @@ public class JsonFormatTest extends TestCase { assertEquals(expectedValues[i], builder.getRepeatedInt64(i)); assertEquals(expectedValues[i], builder.getRepeatedUint64(i)); } - + // Non-integers will still be rejected. assertRejects("optionalInt32", "1.5"); assertRejects("optionalUint32", "1.5"); assertRejects("optionalInt64", "1.5"); assertRejects("optionalUint64", "1.5"); } - + private void assertRejects(String name, String value) { TestAllTypes.Builder builder = TestAllTypes.newBuilder(); try { @@ -312,7 +313,7 @@ public class JsonFormatTest extends TestCase { // Expected. } } - + private void assertAccepts(String name, String value) throws IOException { TestAllTypes.Builder builder = TestAllTypes.newBuilder(); // Both numeric form and string form are accepted. @@ -320,17 +321,17 @@ public class JsonFormatTest extends TestCase { builder.clear(); mergeFromJson("{\"" + name + "\":\"" + value + "\"}", builder); } - + public void testParserRejectOutOfRangeNumericValues() throws Exception { assertAccepts("optionalInt32", String.valueOf(Integer.MAX_VALUE)); assertAccepts("optionalInt32", String.valueOf(Integer.MIN_VALUE)); assertRejects("optionalInt32", String.valueOf(Integer.MAX_VALUE + 1L)); assertRejects("optionalInt32", String.valueOf(Integer.MIN_VALUE - 1L)); - + assertAccepts("optionalUint32", String.valueOf(Integer.MAX_VALUE + 1L)); assertRejects("optionalUint32", "123456789012345"); assertRejects("optionalUint32", "-1"); - + BigInteger one = new BigInteger("1"); BigInteger maxLong = new BigInteger(String.valueOf(Long.MAX_VALUE)); BigInteger minLong = new BigInteger(String.valueOf(Long.MIN_VALUE)); @@ -351,7 +352,7 @@ public class JsonFormatTest extends TestCase { assertAccepts("optionalFloat", String.valueOf(-Float.MAX_VALUE)); assertRejects("optionalFloat", String.valueOf(Double.MAX_VALUE)); assertRejects("optionalFloat", String.valueOf(-Double.MAX_VALUE)); - + BigDecimal moreThanOne = new BigDecimal("1.000001"); BigDecimal maxDouble = new BigDecimal(Double.MAX_VALUE); BigDecimal minDouble = new BigDecimal(-Double.MAX_VALUE); @@ -360,7 +361,7 @@ public class JsonFormatTest extends TestCase { assertRejects("optionalDouble", maxDouble.multiply(moreThanOne).toString()); assertRejects("optionalDouble", minDouble.multiply(moreThanOne).toString()); } - + public void testParserAcceptNull() throws Exception { TestAllTypes.Builder builder = TestAllTypes.newBuilder(); mergeFromJson( @@ -402,7 +403,7 @@ public class JsonFormatTest extends TestCase { + "}", builder); TestAllTypes message = builder.build(); assertEquals(TestAllTypes.getDefaultInstance(), message); - + // Repeated field elements cannot be null. try { builder = TestAllTypes.newBuilder(); @@ -414,7 +415,7 @@ public class JsonFormatTest extends TestCase { } catch (InvalidProtocolBufferException e) { // Exception expected. } - + try { builder = TestAllTypes.newBuilder(); mergeFromJson( @@ -426,14 +427,14 @@ public class JsonFormatTest extends TestCase { // Exception expected. } } - + public void testParserRejectDuplicatedFields() throws Exception { // TODO(xiaofeng): The parser we are currently using (GSON) will accept and keep the last // one if multiple entries have the same name. This is not the desired behavior but it can // only be fixed by using our own parser. Here we only test the cases where the names are // different but still referring to the same field. - - // Duplicated optional fields. + + // Duplicated optional fields. try { TestAllTypes.Builder builder = TestAllTypes.newBuilder(); mergeFromJson( @@ -445,7 +446,7 @@ public class JsonFormatTest extends TestCase { } catch (InvalidProtocolBufferException e) { // Exception expected. } - + // Duplicated repeated fields. try { TestAllTypes.Builder builder = TestAllTypes.newBuilder(); @@ -458,7 +459,7 @@ public class JsonFormatTest extends TestCase { } catch (InvalidProtocolBufferException e) { // Exception expected. } - + // Duplicated oneof fields. try { TestOneof.Builder builder = TestOneof.newBuilder(); @@ -472,7 +473,7 @@ public class JsonFormatTest extends TestCase { // Exception expected. } } - + public void testMapFields() throws Exception { TestMap.Builder builder = TestMap.newBuilder(); builder.getMutableInt32ToInt32Map().put(1, 10); @@ -507,7 +508,7 @@ public class JsonFormatTest extends TestCase { 8, NestedMessage.newBuilder().setValue(1234).build()); builder.getMutableInt32ToEnumMap().put(9, NestedEnum.BAR); TestMap message = builder.build(); - + assertEquals( "{\n" + " \"int32ToInt32Map\": {\n" @@ -598,13 +599,13 @@ public class JsonFormatTest extends TestCase { + " }\n" + "}", toJsonString(message)); assertRoundTripEquals(message); - + // Test multiple entries. builder = TestMap.newBuilder(); builder.getMutableInt32ToInt32Map().put(1, 2); builder.getMutableInt32ToInt32Map().put(3, 4); message = builder.build(); - + assertEquals( "{\n" + " \"int32ToInt32Map\": {\n" @@ -614,7 +615,7 @@ public class JsonFormatTest extends TestCase { + "}", toJsonString(message)); assertRoundTripEquals(message); } - + public void testMapNullValueIsRejected() throws Exception { try { TestMap.Builder builder = TestMap.newBuilder(); @@ -627,7 +628,7 @@ public class JsonFormatTest extends TestCase { } catch (InvalidProtocolBufferException e) { // Exception expected. } - + try { TestMap.Builder builder = TestMap.newBuilder(); mergeFromJson( @@ -640,7 +641,7 @@ public class JsonFormatTest extends TestCase { // Exception expected. } } - + public void testParserAcceptNonQuotedObjectKey() throws Exception { TestMap.Builder builder = TestMap.newBuilder(); mergeFromJson( @@ -652,7 +653,7 @@ public class JsonFormatTest extends TestCase { assertEquals(2, message.getInt32ToInt32Map().get(1).intValue()); assertEquals(3, message.getStringToInt32Map().get("hello").intValue()); } - + public void testWrappers() throws Exception { TestWrappers.Builder builder = TestWrappers.newBuilder(); builder.getBoolValueBuilder().setValue(false); @@ -665,7 +666,7 @@ public class JsonFormatTest extends TestCase { builder.getStringValueBuilder().setValue(""); builder.getBytesValueBuilder().setValue(ByteString.EMPTY); TestWrappers message = builder.build(); - + assertEquals( "{\n" + " \"int32Value\": 0,\n" @@ -691,7 +692,7 @@ public class JsonFormatTest extends TestCase { builder.getStringValueBuilder().setValue("7"); builder.getBytesValueBuilder().setValue(ByteString.copyFrom(new byte[]{8})); message = builder.build(); - + assertEquals( "{\n" + " \"int32Value\": 1,\n" @@ -706,43 +707,43 @@ public class JsonFormatTest extends TestCase { + "}", toJsonString(message)); assertRoundTripEquals(message); } - + public void testTimestamp() throws Exception { TestTimestamp message = TestTimestamp.newBuilder() .setTimestampValue(TimeUtil.parseTimestamp("1970-01-01T00:00:00Z")) .build(); - + assertEquals( "{\n" + " \"timestampValue\": \"1970-01-01T00:00:00Z\"\n" + "}", toJsonString(message)); assertRoundTripEquals(message); } - + public void testDuration() throws Exception { TestDuration message = TestDuration.newBuilder() .setDurationValue(TimeUtil.parseDuration("12345s")) .build(); - + assertEquals( "{\n" + " \"durationValue\": \"12345s\"\n" + "}", toJsonString(message)); assertRoundTripEquals(message); } - + public void testFieldMask() throws Exception { TestFieldMask message = TestFieldMask.newBuilder() .setFieldMaskValue(FieldMaskUtil.fromString("foo.bar,baz")) .build(); - + assertEquals( "{\n" + " \"fieldMaskValue\": \"foo.bar,baz\"\n" + "}", toJsonString(message)); assertRoundTripEquals(message); } - + public void testStruct() throws Exception { // Build a struct with all possible values. TestStruct.Builder builder = TestStruct.newBuilder(); @@ -764,7 +765,7 @@ public class JsonFormatTest extends TestCase { structBuilder.getMutableFields().put( "list_value", Value.newBuilder().setListValue(listBuilder.build()).build()); TestStruct message = builder.build(); - + assertEquals( "{\n" + " \"structValue\": {\n" @@ -778,7 +779,7 @@ public class JsonFormatTest extends TestCase { + " }\n" + "}", toJsonString(message)); assertRoundTripEquals(message); - + builder = TestStruct.newBuilder(); builder.setValue(Value.newBuilder().setNullValueValue(0).build()); message = builder.build(); @@ -787,12 +788,23 @@ public class JsonFormatTest extends TestCase { + " \"value\": null\n" + "}", toJsonString(message)); assertRoundTripEquals(message); + + builder = TestStruct.newBuilder(); + listBuilder = builder.getListValueBuilder(); + listBuilder.addValues(Value.newBuilder().setNumberValue(31831.125).build()); + listBuilder.addValues(Value.newBuilder().setNullValueValue(0).build()); + message = builder.build(); + assertEquals( + "{\n" + + " \"listValue\": [31831.125, null]\n" + + "}", toJsonString(message)); + assertRoundTripEquals(message); } - + public void testAnyFields() throws Exception { TestAllTypes content = TestAllTypes.newBuilder().setOptionalInt32(1234).build(); TestAny message = TestAny.newBuilder().setAnyValue(Any.pack(content)).build(); - + // A TypeRegistry must be provided in order to convert Any types. try { toJsonString(message); @@ -800,11 +812,11 @@ public class JsonFormatTest extends TestCase { } catch (IOException e) { // Expected. } - + JsonFormat.TypeRegistry registry = JsonFormat.TypeRegistry.newBuilder() .add(TestAllTypes.getDescriptor()).build(); JsonFormat.Printer printer = JsonFormat.printer().usingTypeRegistry(registry); - + assertEquals( "{\n" + " \"anyValue\": {\n" @@ -813,8 +825,8 @@ public class JsonFormatTest extends TestCase { + " }\n" + "}" , printer.print(message)); assertRoundTripEquals(message, registry); - - + + // Well-known types have a special formatting when embedded in Any. // // 1. Any in Any. @@ -828,7 +840,7 @@ public class JsonFormatTest extends TestCase { + " }\n" + "}", printer.print(anyMessage)); assertRoundTripEquals(anyMessage, registry); - + // 2. Wrappers in Any. anyMessage = Any.pack(Int32Value.newBuilder().setValue(12345).build()); assertEquals( @@ -894,7 +906,7 @@ public class JsonFormatTest extends TestCase { + " \"value\": \"AQI=\"\n" + "}", printer.print(anyMessage)); assertRoundTripEquals(anyMessage, registry); - + // 3. Timestamp in Any. anyMessage = Any.pack(TimeUtil.parseTimestamp("1969-12-31T23:59:59Z")); assertEquals( @@ -903,7 +915,7 @@ public class JsonFormatTest extends TestCase { + " \"value\": \"1969-12-31T23:59:59Z\"\n" + "}", printer.print(anyMessage)); assertRoundTripEquals(anyMessage, registry); - + // 4. Duration in Any anyMessage = Any.pack(TimeUtil.parseDuration("12345.10s")); assertEquals( @@ -945,7 +957,7 @@ public class JsonFormatTest extends TestCase { + "}", printer.print(anyMessage)); assertRoundTripEquals(anyMessage, registry); } - + public void testParserMissingTypeUrl() throws Exception { try { Any.Builder builder = Any.newBuilder(); @@ -958,7 +970,7 @@ public class JsonFormatTest extends TestCase { // Expected. } } - + public void testParserUnexpectedTypeUrl() throws Exception { try { TestAllTypes.Builder builder = TestAllTypes.newBuilder(); @@ -970,9 +982,9 @@ public class JsonFormatTest extends TestCase { fail("Exception is expected."); } catch (IOException e) { // Expected. - } + } } - + public void testParserRejectTrailingComma() throws Exception { try { TestAllTypes.Builder builder = TestAllTypes.newBuilder(); @@ -1000,13 +1012,13 @@ public class JsonFormatTest extends TestCase { // // Expected. // } } - + public void testParserRejectInvalidBase64() throws Exception { assertRejects("optionalBytes", "!@#$"); // We use standard BASE64 with paddings. assertRejects("optionalBytes", "AQI"); } - + public void testParserRejectInvalidEnumValue() throws Exception { try { TestAllTypes.Builder builder = TestAllTypes.newBuilder(); @@ -1017,7 +1029,7 @@ public class JsonFormatTest extends TestCase { fail("Exception is expected."); } catch (InvalidProtocolBufferException e) { // Expected. - } + } } public void testCustomJsonName() throws Exception { @@ -1026,6 +1038,12 @@ public class JsonFormatTest extends TestCase { assertRoundTripEquals(message); } + public void testDefaultGsonDoesNotHtmlEscape() throws Exception { + TestAllTypes message = TestAllTypes.newBuilder().setOptionalString("=").build(); + assertEquals( + "{\n" + " \"optionalString\": \"=\"" + "\n}", JsonFormat.printer().print(message)); + } + public void testIncludingDefaultValueFields() throws Exception { TestAllTypes message = TestAllTypes.getDefaultInstance(); assertEquals("{\n}", JsonFormat.printer().print(message)); diff --git a/java/util/src/test/proto/com/google/protobuf/util/json_test.proto b/java/util/src/test/proto/com/google/protobuf/util/json_test.proto index 509c1d69..686edc42 100644 --- a/java/util/src/test/proto/com/google/protobuf/util/json_test.proto +++ b/java/util/src/test/proto/com/google/protobuf/util/json_test.proto @@ -28,6 +28,36 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + syntax = "proto3"; package json_test; @@ -159,6 +189,7 @@ message TestFieldMask { message TestStruct { google.protobuf.Struct struct_value = 1; google.protobuf.Value value = 2; + google.protobuf.ListValue list_value = 3; } message TestAny { |