diff options
Diffstat (limited to 'javanano')
14 files changed, 513 insertions, 105 deletions
diff --git a/javanano/README.md b/javanano/README.md index 7d696aa7..e19b90b1 100644 --- a/javanano/README.md +++ b/javanano/README.md @@ -145,6 +145,7 @@ optional_field_style -> default or accessors enum_style -> c or java ignore_services -> true or false parcelable_messages -> true or false +generate_intdefs -> true or false ``` **java_package=\<file-name\>|\<package-name\>** (no default) @@ -302,6 +303,28 @@ parcelable_messages -> true or false Android-specific option to generate Parcelable messages. +**generate_intdefs={true,false}** (default: false) + Android-specific option to generate @IntDef annotations for enums. + + If turned on, an '@IntDef' annotation (a public @interface) will be + generated for each enum, and every integer parameter and return + value in the generated code meant for this enum will be annotated + with it. This interface is generated with the same name and at the + same place as the enum members' container interfaces described + above under 'enum_style=java', regardless of the enum_style option + used. When this is combined with enum_style=java, the interface + will be both the '@IntDef' annotation and the container of the enum + members; otherwise the interface has an empty body. + + Your app must declare a compile-time dependency on the + android-support-annotations library. + + For more information on how these @IntDef annotations help with + compile-time type safety, see: + https://sites.google.com/a/android.com/tools/tech-docs/support-annotations + and + https://developer.android.com/reference/android/support/annotation/IntDef.html + To use nano protobufs within the Android repo: ---------------------------------------------- diff --git a/javanano/pom.xml b/javanano/pom.xml index 7a2be9df..3d8cfb9f 100644 --- a/javanano/pom.xml +++ b/javanano/pom.xml @@ -97,19 +97,19 @@ <arg value="src/test/java/com/google/protobuf/nano/map_test.proto" /> </exec> <exec executable="../src/protoc"> - <arg value="--javanano_out=store_unknown_fields=true,generate_equals=true:target/generated-test-sources" /> + <arg value="--javanano_out=store_unknown_fields=true,generate_equals=true,generate_clone=true:target/generated-test-sources" /> <arg value="--proto_path=src/test/java/com" /> <arg value="src/test/java/com/google/protobuf/nano/unittest_extension_nano.proto" /> <arg value="src/test/java/com/google/protobuf/nano/unittest_extension_singular_nano.proto" /> <arg value="src/test/java/com/google/protobuf/nano/unittest_extension_repeated_nano.proto" /> </exec> <exec executable="../src/protoc"> - <arg value="--javanano_out=store_unknown_fields=true:target/generated-test-sources" /> + <arg value="--javanano_out=store_unknown_fields=true,generate_clone=true:target/generated-test-sources" /> <arg value="--proto_path=src/test/java/com" /> <arg value="src/test/java/com/google/protobuf/nano/unittest_extension_packed_nano.proto" /> </exec> <exec executable="../src/protoc"> - <arg value="--javanano_out=java_nano_generate_has=true,generate_equals=true:target/generated-test-sources" /> + <arg value="--javanano_out=java_nano_generate_has=true,generate_equals=true,generate_clone=true:target/generated-test-sources" /> <arg value="--proto_path=src/test/java/com" /> <arg value="src/test/java/com/google/protobuf/nano/unittest_has_nano.proto" /> </exec> @@ -139,6 +139,15 @@ <arg value="--proto_path=src/test/java/com" /> <arg value="src/test/java/com/google/protobuf/nano/unittest_reference_types_nano.proto" /> </exec> + <exec executable="../src/protoc"> + <arg value="--javanano_out= + optional_field_style=reftypes_compat_mode, + generate_equals=true, + java_outer_classname=google/protobuf/nano/unittest_reference_types_nano.proto|NanoReferenceTypesCompat + :target/generated-test-sources" /> + <arg value="--proto_path=src/test/java/com" /> + <arg value="src/test/java/com/google/protobuf/nano/unittest_reference_types_nano.proto" /> + </exec> </tasks> <testSourceRoot>target/generated-test-sources</testSourceRoot> </configuration> diff --git a/javanano/src/main/java/com/google/protobuf/nano/CodedInputByteBufferNano.java b/javanano/src/main/java/com/google/protobuf/nano/CodedInputByteBufferNano.java index b4f20fde..f3993155 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/CodedInputByteBufferNano.java +++ b/javanano/src/main/java/com/google/protobuf/nano/CodedInputByteBufferNano.java @@ -190,12 +190,12 @@ public final class CodedInputByteBufferNano { if (size <= (bufferSize - bufferPos) && size > 0) { // Fast path: We already have the bytes in a contiguous buffer, so // just copy directly from it. - final String result = new String(buffer, bufferPos, size, "UTF-8"); + final String result = new String(buffer, bufferPos, size, InternalNano.UTF_8); bufferPos += size; return result; } else { // Slow path: Build a byte array first then copy it. - return new String(readRawBytes(size), "UTF-8"); + return new String(readRawBytes(size), InternalNano.UTF_8); } } @@ -236,6 +236,8 @@ public final class CodedInputByteBufferNano { System.arraycopy(buffer, bufferPos, result, 0, size); bufferPos += size; return result; + } else if (size == 0) { + return WireFormatNano.EMPTY_BYTES; } else { // Slow path: Build a byte array first then copy it. return readRawBytes(size); diff --git a/javanano/src/main/java/com/google/protobuf/nano/CodedOutputByteBufferNano.java b/javanano/src/main/java/com/google/protobuf/nano/CodedOutputByteBufferNano.java index 2777f34c..b1b0c53a 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/CodedOutputByteBufferNano.java +++ b/javanano/src/main/java/com/google/protobuf/nano/CodedOutputByteBufferNano.java @@ -31,7 +31,9 @@ package com.google.protobuf.nano; import java.io.IOException; -import java.io.UnsupportedEncodingException; +import java.nio.BufferOverflowException; +import java.nio.ByteBuffer; +import java.nio.ReadOnlyBufferException; /** * Encodes and writes protocol message fields. @@ -48,15 +50,17 @@ import java.io.UnsupportedEncodingException; * @author kneton@google.com Kenton Varda */ public final class CodedOutputByteBufferNano { - private final byte[] buffer; - private final int limit; - private int position; + /* max bytes per java UTF-16 char in UTF-8 */ + private static final int MAX_UTF8_EXPANSION = 3; + private final ByteBuffer buffer; private CodedOutputByteBufferNano(final byte[] buffer, final int offset, final int length) { + this(ByteBuffer.wrap(buffer, offset, length)); + } + + private CodedOutputByteBufferNano(final ByteBuffer buffer) { this.buffer = buffer; - position = offset; - limit = offset + length; } /** @@ -288,14 +292,213 @@ public final class CodedOutputByteBufferNano { /** Write a {@code string} field to the stream. */ public void writeStringNoTag(final String value) throws IOException { - // Unfortunately there does not appear to be any way to tell Java to encode - // UTF-8 directly into our buffer, so we have to let it create its own byte - // array and then copy. - final byte[] bytes = value.getBytes("UTF-8"); - writeRawVarint32(bytes.length); - writeRawBytes(bytes); + // UTF-8 byte length of the string is at least its UTF-16 code unit length (value.length()), + // and at most 3 times of it. Optimize for the case where we know this length results in a + // constant varint length - saves measuring length of the string. + try { + final int minLengthVarIntSize = computeRawVarint32Size(value.length()); + final int maxLengthVarIntSize = computeRawVarint32Size(value.length() * MAX_UTF8_EXPANSION); + if (minLengthVarIntSize == maxLengthVarIntSize) { + int oldPosition = buffer.position(); + // Buffer.position, when passed a position that is past its limit, throws + // IllegalArgumentException, and this class is documented to throw + // OutOfSpaceException instead. + if (buffer.remaining() < minLengthVarIntSize) { + throw new OutOfSpaceException(oldPosition + minLengthVarIntSize, buffer.limit()); + } + buffer.position(oldPosition + minLengthVarIntSize); + encode(value, buffer); + int newPosition = buffer.position(); + buffer.position(oldPosition); + writeRawVarint32(newPosition - oldPosition - minLengthVarIntSize); + buffer.position(newPosition); + } else { + writeRawVarint32(encodedLength(value)); + encode(value, buffer); + } + } catch (BufferOverflowException e) { + final OutOfSpaceException outOfSpaceException = new OutOfSpaceException(buffer.position(), + buffer.limit()); + outOfSpaceException.initCause(e); + throw outOfSpaceException; + } + } + + // These UTF-8 handling methods are copied from Guava's Utf8 class. + /** + * Returns the number of bytes in the UTF-8-encoded form of {@code sequence}. For a string, + * this method is equivalent to {@code string.getBytes(UTF_8).length}, but is more efficient in + * both time and space. + * + * @throws IllegalArgumentException if {@code sequence} contains ill-formed UTF-16 (unpaired + * surrogates) + */ + private static int encodedLength(CharSequence sequence) { + // Warning to maintainers: this implementation is highly optimized. + int utf16Length = sequence.length(); + int utf8Length = utf16Length; + int i = 0; + + // This loop optimizes for pure ASCII. + while (i < utf16Length && sequence.charAt(i) < 0x80) { + i++; + } + + // This loop optimizes for chars less than 0x800. + for (; i < utf16Length; i++) { + char c = sequence.charAt(i); + if (c < 0x800) { + utf8Length += ((0x7f - c) >>> 31); // branch free! + } else { + utf8Length += encodedLengthGeneral(sequence, i); + break; + } + } + + if (utf8Length < utf16Length) { + // Necessary and sufficient condition for overflow because of maximum 3x expansion + throw new IllegalArgumentException("UTF-8 length does not fit in int: " + + (utf8Length + (1L << 32))); + } + return utf8Length; + } + + private static int encodedLengthGeneral(CharSequence sequence, int start) { + int utf16Length = sequence.length(); + int utf8Length = 0; + for (int i = start; i < utf16Length; i++) { + char c = sequence.charAt(i); + if (c < 0x800) { + utf8Length += (0x7f - c) >>> 31; // branch free! + } else { + utf8Length += 2; + // jdk7+: if (Character.isSurrogate(c)) { + if (Character.MIN_SURROGATE <= c && c <= Character.MAX_SURROGATE) { + // Check that we have a well-formed surrogate pair. + int cp = Character.codePointAt(sequence, i); + if (cp < Character.MIN_SUPPLEMENTARY_CODE_POINT) { + throw new IllegalArgumentException("Unpaired surrogate at index " + i); + } + i++; + } + } + } + return utf8Length; + } + + /** + * Encodes {@code sequence} into UTF-8, in {@code byteBuffer}. For a string, this method is + * equivalent to {@code buffer.put(string.getBytes(UTF_8))}, but is more efficient in both time + * and space. Bytes are written starting at the current position. This method requires paired + * surrogates, and therefore does not support chunking. + * + * <p>To ensure sufficient space in the output buffer, either call {@link #encodedLength} to + * compute the exact amount needed, or leave room for {@code 3 * sequence.length()}, which is the + * largest possible number of bytes that any input can be encoded to. + * + * @throws IllegalArgumentException if {@code sequence} contains ill-formed UTF-16 (unpaired + * surrogates) + * @throws BufferOverflowException if {@code sequence} encoded in UTF-8 does not fit in + * {@code byteBuffer}'s remaining space. + * @throws ReadOnlyBufferException if {@code byteBuffer} is a read-only buffer. + */ + private static void encode(CharSequence sequence, ByteBuffer byteBuffer) { + if (byteBuffer.isReadOnly()) { + throw new ReadOnlyBufferException(); + } else if (byteBuffer.hasArray()) { + try { + int encoded = encode(sequence, + byteBuffer.array(), + byteBuffer.arrayOffset() + byteBuffer.position(), + byteBuffer.remaining()); + byteBuffer.position(encoded - byteBuffer.arrayOffset()); + } catch (ArrayIndexOutOfBoundsException e) { + BufferOverflowException boe = new BufferOverflowException(); + boe.initCause(e); + throw boe; + } + } else { + encodeDirect(sequence, byteBuffer); + } } + private static void encodeDirect(CharSequence sequence, ByteBuffer byteBuffer) { + int utf16Length = sequence.length(); + for (int i = 0; i < utf16Length; i++) { + final char c = sequence.charAt(i); + if (c < 0x80) { // ASCII + byteBuffer.put((byte) c); + } else if (c < 0x800) { // 11 bits, two UTF-8 bytes + byteBuffer.put((byte) ((0xF << 6) | (c >>> 6))); + byteBuffer.put((byte) (0x80 | (0x3F & c))); + } else if (c < Character.MIN_SURROGATE || Character.MAX_SURROGATE < c) { + // Maximium single-char code point is 0xFFFF, 16 bits, three UTF-8 bytes + byteBuffer.put((byte) ((0xF << 5) | (c >>> 12))); + byteBuffer.put((byte) (0x80 | (0x3F & (c >>> 6)))); + byteBuffer.put((byte) (0x80 | (0x3F & c))); + } else { + final char low; + if (i + 1 == sequence.length() + || !Character.isSurrogatePair(c, (low = sequence.charAt(++i)))) { + throw new IllegalArgumentException("Unpaired surrogate at index " + (i - 1)); + } + int codePoint = Character.toCodePoint(c, low); + byteBuffer.put((byte) ((0xF << 4) | (codePoint >>> 18))); + byteBuffer.put((byte) (0x80 | (0x3F & (codePoint >>> 12)))); + byteBuffer.put((byte) (0x80 | (0x3F & (codePoint >>> 6)))); + byteBuffer.put((byte) (0x80 | (0x3F & codePoint))); + } + } + } + + private static int encode(CharSequence sequence, byte[] bytes, int offset, int length) { + int utf16Length = sequence.length(); + int j = offset; + int i = 0; + int limit = offset + length; + // Designed to take advantage of + // https://wikis.oracle.com/display/HotSpotInternals/RangeCheckElimination + for (char c; i < utf16Length && i + j < limit && (c = sequence.charAt(i)) < 0x80; i++) { + bytes[j + i] = (byte) c; + } + if (i == utf16Length) { + return j + utf16Length; + } + j += i; + for (char c; i < utf16Length; i++) { + c = sequence.charAt(i); + if (c < 0x80 && j < limit) { + bytes[j++] = (byte) c; + } else if (c < 0x800 && j <= limit - 2) { // 11 bits, two UTF-8 bytes + bytes[j++] = (byte) ((0xF << 6) | (c >>> 6)); + bytes[j++] = (byte) (0x80 | (0x3F & c)); + } else if ((c < Character.MIN_SURROGATE || Character.MAX_SURROGATE < c) && j <= limit - 3) { + // Maximum single-char code point is 0xFFFF, 16 bits, three UTF-8 bytes + bytes[j++] = (byte) ((0xF << 5) | (c >>> 12)); + bytes[j++] = (byte) (0x80 | (0x3F & (c >>> 6))); + bytes[j++] = (byte) (0x80 | (0x3F & c)); + } else if (j <= limit - 4) { + // Minimum code point represented by a surrogate pair is 0x10000, 17 bits, four UTF-8 bytes + final char low; + if (i + 1 == sequence.length() + || !Character.isSurrogatePair(c, (low = sequence.charAt(++i)))) { + throw new IllegalArgumentException("Unpaired surrogate at index " + (i - 1)); + } + int codePoint = Character.toCodePoint(c, low); + bytes[j++] = (byte) ((0xF << 4) | (codePoint >>> 18)); + bytes[j++] = (byte) (0x80 | (0x3F & (codePoint >>> 12))); + bytes[j++] = (byte) (0x80 | (0x3F & (codePoint >>> 6))); + bytes[j++] = (byte) (0x80 | (0x3F & codePoint)); + } else { + throw new ArrayIndexOutOfBoundsException("Failed writing " + c + " at index " + j); + } + } + return j; + } + + // End guava UTF-8 methods + + /** Write a {@code group} field to the stream. */ public void writeGroupNoTag(final MessageNano value) throws IOException { value.writeTo(this); @@ -603,13 +806,8 @@ public final class CodedOutputByteBufferNano { * {@code string} field. */ public static int computeStringSizeNoTag(final String value) { - try { - final byte[] bytes = value.getBytes("UTF-8"); - return computeRawVarint32Size(bytes.length) + - bytes.length; - } catch (UnsupportedEncodingException e) { - throw new RuntimeException("UTF-8 not supported."); - } + final int length = encodedLength(value); + return computeRawVarint32Size(length) + length; } /** @@ -692,7 +890,7 @@ public final class CodedOutputByteBufferNano { * Otherwise, throws {@code UnsupportedOperationException}. */ public int spaceLeft() { - return limit - position; + return buffer.remaining(); } /** @@ -710,6 +908,23 @@ public final class CodedOutputByteBufferNano { } /** + * Returns the position within the internal buffer. + */ + public int position() { + return buffer.position(); + } + + /** + * Resets the position within the internal buffer to zero. + * + * @see #position + * @see #spaceLeft + */ + public void reset() { + buffer.clear(); + } + + /** * If you create a CodedOutputStream around a simple flat array, you must * not attempt to write more bytes than the array has space. Otherwise, * this exception will be thrown. @@ -725,12 +940,12 @@ public final class CodedOutputByteBufferNano { /** Write a single byte. */ public void writeRawByte(final byte value) throws IOException { - if (position == limit) { + if (!buffer.hasRemaining()) { // We're writing to a single buffer. - throw new OutOfSpaceException(position, limit); + throw new OutOfSpaceException(buffer.position(), buffer.limit()); } - buffer[position++] = value; + buffer.put(value); } /** Write a single byte, represented by an integer value. */ @@ -746,13 +961,11 @@ public final class CodedOutputByteBufferNano { /** Write part of an array of bytes. */ public void writeRawBytes(final byte[] value, int offset, int length) throws IOException { - if (limit - position >= length) { - // We have room in the current buffer. - System.arraycopy(value, offset, buffer, position, length); - position += length; + if (buffer.remaining() >= length) { + buffer.put(value, offset, length); } else { // We're writing to a single buffer. - throw new OutOfSpaceException(position, limit); + throw new OutOfSpaceException(buffer.position(), buffer.limit()); } } diff --git a/javanano/src/main/java/com/google/protobuf/nano/ExtendableMessageNano.java b/javanano/src/main/java/com/google/protobuf/nano/ExtendableMessageNano.java index b979390d..87973d76 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/ExtendableMessageNano.java +++ b/javanano/src/main/java/com/google/protobuf/nano/ExtendableMessageNano.java @@ -160,28 +160,10 @@ public abstract class ExtendableMessageNano<M extends ExtendableMessageNano<M>> return true; } - /** - * Returns whether the stored unknown field data in this message is equivalent to that in the - * other message. - * - * @param other the other message. - * @return whether the two sets of unknown field data are equal. - */ - protected final boolean unknownFieldDataEquals(M other) { - if (unknownFieldData == null || unknownFieldData.isEmpty()) { - return other.unknownFieldData == null || other.unknownFieldData.isEmpty(); - } else { - return unknownFieldData.equals(other.unknownFieldData); - } - } - - /** - * Computes the hashcode representing the unknown field data stored in this message. - * - * @return the hashcode for the unknown field data. - */ - protected final int unknownFieldDataHashCode() { - return (unknownFieldData == null || unknownFieldData.isEmpty() - ? 0 : unknownFieldData.hashCode()); + @Override + public M clone() throws CloneNotSupportedException { + M cloned = (M) super.clone(); + InternalNano.cloneUnknownFieldData(this, cloned); + return cloned; } } diff --git a/javanano/src/main/java/com/google/protobuf/nano/Extension.java b/javanano/src/main/java/com/google/protobuf/nano/Extension.java index c29b030f..c458f9b1 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/Extension.java +++ b/javanano/src/main/java/com/google/protobuf/nano/Extension.java @@ -79,12 +79,30 @@ public class Extension<M extends ExtendableMessageNano<M>, T> { * Should be used by the generated code only. * * @param type {@link #TYPE_MESSAGE} or {@link #TYPE_GROUP} + * @deprecated use {@link #createMessageTyped(int, Class, long)} instead. */ + @Deprecated public static <M extends ExtendableMessageNano<M>, T extends MessageNano> Extension<M, T> createMessageTyped(int type, Class<T> clazz, int tag) { return new Extension<M, T>(type, clazz, tag, false); } + // Note: these create...() methods take a long for the tag parameter, + // because tags are represented as unsigned ints, and these values exist + // in generated code as long values. However, they can fit in 32-bits, so + // it's safe to cast them to int without loss of precision. + + /** + * Creates an {@code Extension} of the given message type and tag number. + * Should be used by the generated code only. + * + * @param type {@link #TYPE_MESSAGE} or {@link #TYPE_GROUP} + */ + public static <M extends ExtendableMessageNano<M>, T extends MessageNano> + Extension<M, T> createMessageTyped(int type, Class<T> clazz, long tag) { + return new Extension<M, T>(type, clazz, (int) tag, false); + } + /** * Creates a repeated {@code Extension} of the given message type and tag number. * Should be used by the generated code only. @@ -92,8 +110,8 @@ public class Extension<M extends ExtendableMessageNano<M>, T> { * @param type {@link #TYPE_MESSAGE} or {@link #TYPE_GROUP} */ public static <M extends ExtendableMessageNano<M>, T extends MessageNano> - Extension<M, T[]> createRepeatedMessageTyped(int type, Class<T[]> clazz, int tag) { - return new Extension<M, T[]>(type, clazz, tag, true); + Extension<M, T[]> createRepeatedMessageTyped(int type, Class<T[]> clazz, long tag) { + return new Extension<M, T[]>(type, clazz, (int) tag, true); } /** @@ -104,8 +122,8 @@ public class Extension<M extends ExtendableMessageNano<M>, T> { * @param clazz the boxed Java type of this extension */ public static <M extends ExtendableMessageNano<M>, T> - Extension<M, T> createPrimitiveTyped(int type, Class<T> clazz, int tag) { - return new PrimitiveExtension<M, T>(type, clazz, tag, false, 0, 0); + Extension<M, T> createPrimitiveTyped(int type, Class<T> clazz, long tag) { + return new PrimitiveExtension<M, T>(type, clazz, (int) tag, false, 0, 0); } /** @@ -117,8 +135,9 @@ public class Extension<M extends ExtendableMessageNano<M>, T> { */ public static <M extends ExtendableMessageNano<M>, T> Extension<M, T> createRepeatedPrimitiveTyped( - int type, Class<T> clazz, int tag, int nonPackedTag, int packedTag) { - return new PrimitiveExtension<M, T>(type, clazz, tag, true, nonPackedTag, packedTag); + int type, Class<T> clazz, long tag, long nonPackedTag, long packedTag) { + return new PrimitiveExtension<M, T>(type, clazz, (int) tag, true, + (int) nonPackedTag, (int) packedTag); } /** @@ -136,7 +155,7 @@ public class Extension<M extends ExtendableMessageNano<M>, T> { protected final Class<T> clazz; /** - * Tag number of this extension. + * Tag number of this extension. The data should be viewed as an unsigned 32-bit value. */ public final int tag; diff --git a/javanano/src/main/java/com/google/protobuf/nano/FieldArray.java b/javanano/src/main/java/com/google/protobuf/nano/FieldArray.java index cdb66da2..eca9c0d9 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/FieldArray.java +++ b/javanano/src/main/java/com/google/protobuf/nano/FieldArray.java @@ -35,9 +35,12 @@ package com.google.protobuf.nano; * A custom version of {@link android.util.SparseArray} with the minimal API * for storing {@link FieldData} objects. * + * <p>This class is an internal implementation detail of nano and should not + * be called directly by clients. + * * Based on {@link android.support.v4.util.SpareArrayCompat}. */ -class FieldArray { +public final class FieldArray implements Cloneable { private static final FieldData DELETED = new FieldData(); private boolean mGarbage = false; @@ -48,7 +51,7 @@ class FieldArray { /** * Creates a new FieldArray containing no fields. */ - public FieldArray() { + FieldArray() { this(10); } @@ -57,7 +60,7 @@ class FieldArray { * require any additional memory allocation to store the specified * number of mappings. */ - public FieldArray(int initialCapacity) { + FieldArray(int initialCapacity) { initialCapacity = idealIntArraySize(initialCapacity); mFieldNumbers = new int[initialCapacity]; mData = new FieldData[initialCapacity]; @@ -68,7 +71,7 @@ class FieldArray { * Gets the FieldData mapped from the specified fieldNumber, or <code>null</code> * if no such mapping has been made. */ - public FieldData get(int fieldNumber) { + FieldData get(int fieldNumber) { int i = binarySearch(fieldNumber); if (i < 0 || mData[i] == DELETED) { @@ -81,7 +84,7 @@ class FieldArray { /** * Removes the data from the specified fieldNumber, if there was any. */ - public void remove(int fieldNumber) { + void remove(int fieldNumber) { int i = binarySearch(fieldNumber); if (i >= 0 && mData[i] != DELETED) { @@ -118,7 +121,7 @@ class FieldArray { * Adds a mapping from the specified fieldNumber to the specified data, * replacing the previous mapping if there was one. */ - public void put(int fieldNumber, FieldData data) { + void put(int fieldNumber, FieldData data) { int i = binarySearch(fieldNumber); if (i >= 0) { @@ -167,7 +170,7 @@ class FieldArray { * Returns the number of key-value mappings that this FieldArray * currently stores. */ - public int size() { + int size() { if (mGarbage) { gc(); } @@ -184,7 +187,7 @@ class FieldArray { * the value from the <code>index</code>th key-value mapping that this * FieldArray stores. */ - public FieldData dataAt(int index) { + FieldData dataAt(int index) { if (mGarbage) { gc(); } @@ -270,4 +273,19 @@ class FieldArray { } return true; } + + @Override + public final FieldArray clone() { + // Trigger GC so we compact and don't copy DELETED elements. + int size = size(); + FieldArray clone = new FieldArray(size); + System.arraycopy(mFieldNumbers, 0, clone.mFieldNumbers, 0, size); + for (int i = 0; i < size; i++) { + if (mData[i] != null) { + clone.mData[i] = mData[i].clone(); + } + } + clone.mSize = size; + return clone; + } } diff --git a/javanano/src/main/java/com/google/protobuf/nano/FieldData.java b/javanano/src/main/java/com/google/protobuf/nano/FieldData.java index 21ead88b..ebebabc8 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/FieldData.java +++ b/javanano/src/main/java/com/google/protobuf/nano/FieldData.java @@ -39,7 +39,7 @@ import java.util.List; * Stores unknown fields. These might be extensions or fields that the generated API doesn't * know about yet. */ -class FieldData { +class FieldData implements Cloneable { private Extension<?, ?> cachedExtension; private Object value; /** The serialised values for this object. Will be cleared if getValue is called */ @@ -187,4 +187,54 @@ class FieldData { return result; } + @Override + public final FieldData clone() { + FieldData clone = new FieldData(); + try { + clone.cachedExtension = cachedExtension; + if (unknownFieldData == null) { + clone.unknownFieldData = null; + } else { + clone.unknownFieldData.addAll(unknownFieldData); + } + + // Whether we need to deep clone value depends on its type. Primitive reference types + // (e.g. Integer, Long etc.) are ok, since they're immutable. We need to clone arrays + // and messages. + if (value == null) { + // No cloning required. + } else if (value instanceof MessageNano) { + clone.value = ((MessageNano) value).clone(); + } else if (value instanceof byte[]) { + clone.value = ((byte[]) value).clone(); + } else if (value instanceof byte[][]) { + byte[][] valueArray = (byte[][]) value; + byte[][] cloneArray = new byte[valueArray.length][]; + clone.value = cloneArray; + for (int i = 0; i < valueArray.length; i++) { + cloneArray[i] = valueArray[i].clone(); + } + } else if (value instanceof boolean[]) { + clone.value = ((boolean[]) value).clone(); + } else if (value instanceof int[]) { + clone.value = ((int[]) value).clone(); + } else if (value instanceof long[]) { + clone.value = ((long[]) value).clone(); + } else if (value instanceof float[]) { + clone.value = ((float[]) value).clone(); + } else if (value instanceof double[]) { + clone.value = ((double[]) value).clone(); + } else if (value instanceof MessageNano[]) { + MessageNano[] valueArray = (MessageNano[]) value; + MessageNano[] cloneArray = new MessageNano[valueArray.length]; + clone.value = cloneArray; + for (int i = 0; i < valueArray.length; i++) { + cloneArray[i] = valueArray[i].clone(); + } + } + return clone; + } catch (CloneNotSupportedException e) { + throw new AssertionError(e); + } + } } diff --git a/javanano/src/main/java/com/google/protobuf/nano/InternalNano.java b/javanano/src/main/java/com/google/protobuf/nano/InternalNano.java index 4a08bb58..f1263df5 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/InternalNano.java +++ b/javanano/src/main/java/com/google/protobuf/nano/InternalNano.java @@ -33,7 +33,7 @@ package com.google.protobuf.nano; import com.google.protobuf.nano.MapFactories.MapFactory; import java.io.IOException; -import java.io.UnsupportedEncodingException; +import java.nio.charset.Charset; import java.util.Arrays; import java.util.Map; import java.util.Map.Entry; @@ -67,6 +67,8 @@ public final class InternalNano { public static final int TYPE_SINT32 = 17; public static final int TYPE_SINT64 = 18; + protected static final Charset UTF_8 = Charset.forName("UTF-8"); + protected static final Charset ISO_8859_1 = Charset.forName("ISO-8859-1"); private InternalNano() {} @@ -111,14 +113,7 @@ public final class InternalNano { * generated code calls this automatically. */ public static String stringDefaultValue(String bytes) { - try { - return new String(bytes.getBytes("ISO-8859-1"), "UTF-8"); - } catch (UnsupportedEncodingException e) { - // This should never happen since all JVMs are required to implement - // both of the above character sets. - throw new IllegalStateException( - "Java VM does not support a standard character set.", e); - } + return new String(bytes.getBytes(ISO_8859_1), InternalNano.UTF_8); } /** @@ -130,14 +125,7 @@ public final class InternalNano { * embed raw bytes as a string literal with ISO-8859-1 encoding. */ public static byte[] bytesDefaultValue(String bytes) { - try { - return bytes.getBytes("ISO-8859-1"); - } catch (UnsupportedEncodingException e) { - // This should never happen since all JVMs are required to implement - // ISO-8859-1. - throw new IllegalStateException( - "Java VM does not support a standard character set.", e); - } + return bytes.getBytes(ISO_8859_1); } /** @@ -145,11 +133,7 @@ public final class InternalNano { * UnsupportedEncodingException to a RuntimeException. */ public static byte[] copyFromUtf8(final String text) { - try { - return text.getBytes("UTF-8"); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException("UTF-8 not supported?"); - } + return text.getBytes(InternalNano.UTF_8); } /** @@ -552,4 +536,12 @@ public final class InternalNano { } return o.hashCode(); } + + // This avoids having to make FieldArray public. + public static void cloneUnknownFieldData(ExtendableMessageNano original, + ExtendableMessageNano cloned) { + if (original.unknownFieldData != null) { + cloned.unknownFieldData = (FieldArray) original.unknownFieldData.clone(); + } + } } diff --git a/javanano/src/main/java/com/google/protobuf/nano/MessageNano.java b/javanano/src/main/java/com/google/protobuf/nano/MessageNano.java index 81e58571..23475027 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/MessageNano.java +++ b/javanano/src/main/java/com/google/protobuf/nano/MessageNano.java @@ -187,4 +187,12 @@ public abstract class MessageNano { public String toString() { return MessageNanoPrinter.print(this); } + + /** + * Provides support for cloning. This only works if you specify the generate_clone method. + */ + @Override + public MessageNano clone() throws CloneNotSupportedException { + return (MessageNano) super.clone(); + } } diff --git a/javanano/src/main/java/com/google/protobuf/nano/MessageNanoPrinter.java b/javanano/src/main/java/com/google/protobuf/nano/MessageNanoPrinter.java index c4b2ad3d..d9500bb9 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/MessageNanoPrinter.java +++ b/javanano/src/main/java/com/google/protobuf/nano/MessageNanoPrinter.java @@ -109,6 +109,10 @@ public final class MessageNanoPrinter { for (Field field : clazz.getFields()) { int modifiers = field.getModifiers(); String fieldName = field.getName(); + if ("cachedSize".equals(fieldName)) { + // TODO(bduff): perhaps cachedSize should have a more obscure name. + continue; + } if ((modifiers & Modifier.PUBLIC) == Modifier.PUBLIC && (modifiers & Modifier.STATIC) != Modifier.STATIC diff --git a/javanano/src/main/java/com/google/protobuf/nano/UnknownFieldData.java b/javanano/src/main/java/com/google/protobuf/nano/UnknownFieldData.java index a17fccf3..b1678d1b 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/UnknownFieldData.java +++ b/javanano/src/main/java/com/google/protobuf/nano/UnknownFieldData.java @@ -42,6 +42,10 @@ import java.util.Arrays; final class UnknownFieldData { final int tag; + /** + * Important: this should be treated as immutable, even though it's possible + * to change the array values. + */ final byte[] bytes; UnknownFieldData(int tag, byte[] bytes) { diff --git a/javanano/src/test/java/com/google/protobuf/nano/NanoTest.java b/javanano/src/test/java/com/google/protobuf/nano/NanoTest.java index 4cc77226..3a75777a 100644 --- a/javanano/src/test/java/com/google/protobuf/nano/NanoTest.java +++ b/javanano/src/test/java/com/google/protobuf/nano/NanoTest.java @@ -31,11 +31,13 @@ package com.google.protobuf.nano; import com.google.protobuf.nano.MapTestProto.TestMap; +import com.google.protobuf.nano.CodedOutputByteBufferNano; import com.google.protobuf.nano.MapTestProto.TestMap.MessageValue; import com.google.protobuf.nano.NanoAccessorsOuterClass.TestNanoAccessors; import com.google.protobuf.nano.NanoHasOuterClass.TestAllTypesNanoHas; import com.google.protobuf.nano.NanoOuterClass.TestAllTypesNano; import com.google.protobuf.nano.UnittestRecursiveNano.RecursiveMessageNano; +import com.google.protobuf.nano.NanoReferenceTypesCompat; import com.google.protobuf.nano.UnittestSimpleNano.SimpleMessageNano; import com.google.protobuf.nano.UnittestSingleNano.SingleMessageNano; import com.google.protobuf.nano.testext.Extensions; @@ -458,7 +460,7 @@ public class NanoTest extends TestCase { assertFalse(msg.optionalBytes.length > 0); msg.optionalBytes = InternalNano.copyFromUtf8("hello"); assertTrue(msg.optionalBytes.length > 0); - assertEquals("hello", new String(msg.optionalBytes, "UTF-8")); + assertEquals("hello", new String(msg.optionalBytes, InternalNano.UTF_8)); msg.clear(); assertFalse(msg.optionalBytes.length > 0); msg.clear() @@ -476,7 +478,7 @@ public class NanoTest extends TestCase { TestAllTypesNano newMsg = TestAllTypesNano.parseFrom(result); assertTrue(newMsg.optionalBytes.length > 0); - assertEquals("bye", new String(newMsg.optionalBytes, "UTF-8")); + assertEquals("bye", new String(newMsg.optionalBytes, InternalNano.UTF_8)); } public void testNanoOptionalGroup() throws Exception { @@ -1346,14 +1348,14 @@ public class NanoTest extends TestCase { InternalNano.copyFromUtf8("bye"), InternalNano.copyFromUtf8("boo") }; - assertEquals("bye", new String(msg.repeatedBytes[1], "UTF-8")); - assertEquals("boo", new String(msg.repeatedBytes[2], "UTF-8")); + assertEquals("bye", new String(msg.repeatedBytes[1], InternalNano.UTF_8)); + assertEquals("boo", new String(msg.repeatedBytes[2], InternalNano.UTF_8)); msg.clear(); assertEquals(0, msg.repeatedBytes.length); msg.clear() .repeatedBytes = new byte[][] { InternalNano.copyFromUtf8("boo") }; assertEquals(1, msg.repeatedBytes.length); - assertEquals("boo", new String(msg.repeatedBytes[0], "UTF-8")); + assertEquals("boo", new String(msg.repeatedBytes[0], InternalNano.UTF_8)); msg.clear(); assertEquals(0, msg.repeatedBytes.length); @@ -1385,8 +1387,8 @@ public class NanoTest extends TestCase { newMsg = TestAllTypesNano.parseFrom(result); assertEquals(2, newMsg.repeatedBytes.length); - assertEquals("hello", new String(newMsg.repeatedBytes[0], "UTF-8")); - assertEquals("world", new String(newMsg.repeatedBytes[1], "UTF-8")); + assertEquals("hello", new String(newMsg.repeatedBytes[0], InternalNano.UTF_8)); + assertEquals("world", new String(newMsg.repeatedBytes[1], InternalNano.UTF_8)); } public void testNanoRepeatedGroup() throws Exception { @@ -2277,9 +2279,9 @@ public class NanoTest extends TestCase { assertTrue(52.0e3 == msg.defaultDouble); assertEquals(true, msg.defaultBool); assertEquals("hello", msg.defaultString); - assertEquals("world", new String(msg.defaultBytes, "UTF-8")); + assertEquals("world", new String(msg.defaultBytes, InternalNano.UTF_8)); assertEquals("dünya", msg.defaultStringNonascii); - assertEquals("dünyab", new String(msg.defaultBytesNonascii, "UTF-8")); + assertEquals("dünyab", new String(msg.defaultBytesNonascii, InternalNano.UTF_8)); assertEquals(TestAllTypesNano.BAR, msg.defaultNestedEnum); assertEquals(NanoOuterClass.FOREIGN_NANO_BAR, msg.defaultForeignEnum); assertEquals(UnittestImportNano.IMPORT_NANO_BAR, msg.defaultImportEnum); @@ -2300,6 +2302,59 @@ public class NanoTest extends TestCase { } } + public void testDifferentStringLengthsNano() throws Exception { + // Test string serialization roundtrip using strings of the following lengths, + // with ASCII and Unicode characters requiring different UTF-8 byte counts per + // char, hence causing the length delimiter varint to sometimes require more + // bytes for the Unicode strings than the ASCII string of the same length. + int[] lengths = new int[] { + 0, + 1, + (1 << 4) - 1, // 1 byte for ASCII and Unicode + (1 << 7) - 1, // 1 byte for ASCII, 2 bytes for Unicode + (1 << 11) - 1, // 2 bytes for ASCII and Unicode + (1 << 14) - 1, // 2 bytes for ASCII, 3 bytes for Unicode + (1 << 17) - 1, // 3 bytes for ASCII and Unicode + }; + for (int i : lengths) { + testEncodingOfString('q', i); // 1 byte per char + testEncodingOfString('\u07FF', i); // 2 bytes per char + testEncodingOfString('\u0981', i); // 3 bytes per char + } + } + + /** Regression test for https://github.com/google/protobuf/issues/292 */ + public void testCorrectExceptionThrowWhenEncodingStringsWithoutEnoughSpace() throws Exception { + String testCase = "Foooooooo"; + assertEquals(CodedOutputByteBufferNano.computeRawVarint32Size(testCase.length()), + CodedOutputByteBufferNano.computeRawVarint32Size(testCase.length() * 3)); + assertEquals(11, CodedOutputByteBufferNano.computeStringSize(1, testCase)); + // Tag is one byte, varint describing string length is 1 byte, string length is 9 bytes. + // An array of size 1 will cause a failure when trying to write the varint. + for (int i = 0; i < 11; i++) { + CodedOutputByteBufferNano bufferNano = CodedOutputByteBufferNano.newInstance(new byte[i]); + try { + bufferNano.writeString(1, testCase); + fail("Should have thrown an out of space exception"); + } catch (CodedOutputByteBufferNano.OutOfSpaceException expected) {} + } + } + + private void testEncodingOfString(char c, int length) throws InvalidProtocolBufferNanoException { + TestAllTypesNano testAllTypesNano = new TestAllTypesNano(); + final String fullString = fullString(c, length); + testAllTypesNano.optionalString = fullString; + final TestAllTypesNano resultNano = new TestAllTypesNano(); + MessageNano.mergeFrom(resultNano, MessageNano.toByteArray(testAllTypesNano)); + assertEquals(fullString, resultNano.optionalString); + } + + private String fullString(char c, int length) { + char[] result = new char[length]; + Arrays.fill(result, c); + return new String(result); + } + public void testNanoWithHasParseFrom() throws Exception { TestAllTypesNanoHas msg = null; // Test false on creation, after clear and upon empty parse. @@ -2385,7 +2440,7 @@ public class NanoTest extends TestCase { assertEquals(TestAllTypesNanoHas.FOO, newMsg.optionalNestedEnum); assertEquals(41, newMsg.defaultInt32); assertEquals("hello", newMsg.defaultString); - assertEquals("world", new String(newMsg.defaultBytes, "UTF-8")); + assertEquals("world", new String(newMsg.defaultBytes, InternalNano.UTF_8)); assertEquals(TestAllTypesNanoHas.BAR, newMsg.defaultNestedEnum); assertEquals(Float.NaN, newMsg.defaultFloatNan); assertEquals(0, newMsg.id); @@ -2567,7 +2622,7 @@ public class NanoTest extends TestCase { assertEquals(TestNanoAccessors.FOO, newMsg.getOptionalNestedEnum()); assertEquals(41, newMsg.getDefaultInt32()); assertEquals("hello", newMsg.getDefaultString()); - assertEquals("world", new String(newMsg.getDefaultBytes(), "UTF-8")); + assertEquals("world", new String(newMsg.getDefaultBytes(), InternalNano.UTF_8)); assertEquals(TestNanoAccessors.BAR, newMsg.getDefaultNestedEnum()); assertEquals(Float.NaN, newMsg.getDefaultFloatNan()); assertEquals(0, newMsg.id); @@ -2986,6 +3041,10 @@ public class NanoTest extends TestCase { assertTrue(Arrays.equals(floats, message.getExtension(RepeatedExtensions.repeatedFloat))); assertTrue(Arrays.equals(doubles, message.getExtension(RepeatedExtensions.repeatedDouble))); assertTrue(Arrays.equals(enums, message.getExtension(RepeatedExtensions.repeatedEnum))); + + // Clone the message and ensure it's still equal. + Extensions.ExtendableMessage clone = message.clone(); + assertEquals(clone, message); } public void testNullExtensions() throws Exception { @@ -4345,6 +4404,11 @@ public class NanoTest extends TestCase { assertMapSet(testMap.sfixed64ToSfixed64Field, int64Values, int64Values); } + public void testRepeatedFieldInitializedInReftypesCompatMode() { + NanoReferenceTypesCompat.TestAllTypesNano proto = new NanoReferenceTypesCompat.TestAllTypesNano(); + assertNotNull(proto.repeatedString); + } + private void assertRepeatedPackablesEqual( NanoRepeatedPackables.NonPacked nonPacked, NanoRepeatedPackables.Packed packed) { // Not using MessageNano.equals() -- that belongs to a separate test. @@ -4364,6 +4428,22 @@ public class NanoTest extends TestCase { assertTrue(Arrays.equals(nonPacked.enums, packed.enums)); } + public void testClone() throws Exception { + // A simple message. + AnotherMessage anotherMessage = new AnotherMessage(); + anotherMessage.string = "Hello"; + anotherMessage.value = true; + anotherMessage.integers = new int[] { 1, 2, 3 }; + + AnotherMessage clone = anotherMessage.clone(); + assertEquals(clone, anotherMessage); + + // Verify it was a deep clone - changes to the clone shouldn't affect the + // original. + clone.integers[1] = 100; + assertFalse(clone.equals(anotherMessage)); + } + private void assertHasWireData(MessageNano message, boolean expected) { byte[] bytes = MessageNano.toByteArray(message); int wireLength = bytes.length; diff --git a/javanano/src/test/java/com/google/protobuf/nano/unittest_extension_nano.proto b/javanano/src/test/java/com/google/protobuf/nano/unittest_extension_nano.proto index 2a678a80..ca56b3dd 100644 --- a/javanano/src/test/java/com/google/protobuf/nano/unittest_extension_nano.proto +++ b/javanano/src/test/java/com/google/protobuf/nano/unittest_extension_nano.proto @@ -16,11 +16,15 @@ enum AnEnum { message AnotherMessage { optional string string = 1; optional bool value = 2; + repeated int32 integers = 3; } message ContainerMessage { extend ExtendableMessage { optional bool another_thing = 100; + // The largest permitted field number, per + // https://developers.google.com/protocol-buffers/docs/proto#simple + optional bool large_field_number = 536870911; } } |