From e841bac4fcf47f809e089a70d5f84ac37b3883df Mon Sep 17 00:00:00 2001 From: Feng Xiao Date: Fri, 11 Dec 2015 17:09:20 -0800 Subject: Down-integrate from internal code base. --- .../java/com/google/protobuf/AbstractMessage.java | 13 +- .../java/com/google/protobuf/BooleanArrayList.java | 11 +- .../com/google/protobuf/BoundedByteString.java | 68 +---- .../main/java/com/google/protobuf/ByteString.java | 237 +++++++++++----- .../java/com/google/protobuf/CodedInputStream.java | 153 +++++----- .../com/google/protobuf/CodedOutputStream.java | 91 ++---- .../main/java/com/google/protobuf/Descriptors.java | 43 +++ .../java/com/google/protobuf/DoubleArrayList.java | 11 +- .../java/com/google/protobuf/FloatArrayList.java | 11 +- .../com/google/protobuf/GeneratedMessageLite.java | 31 ++- .../java/com/google/protobuf/IntArrayList.java | 11 +- .../com/google/protobuf/LiteralByteString.java | 213 ++++---------- .../java/com/google/protobuf/LongArrayList.java | 11 +- .../java/com/google/protobuf/MapFieldLite.java | 9 +- .../com/google/protobuf/MessageLiteToString.java | 0 .../java/com/google/protobuf/NioByteString.java | 309 +++++++++++++++++++++ .../com/google/protobuf/ProtobufArrayList.java | 4 + .../java/com/google/protobuf/RopeByteString.java | 252 ++++++----------- .../main/java/com/google/protobuf/TextFormat.java | 54 +++- .../com/google/protobuf/TextFormatEscaper.java | 0 .../com/google/protobuf/UnsafeByteStrings.java | 55 ++++ 21 files changed, 960 insertions(+), 627 deletions(-) create mode 100644 java/src/main/java/com/google/protobuf/MessageLiteToString.java create mode 100644 java/src/main/java/com/google/protobuf/NioByteString.java create mode 100644 java/src/main/java/com/google/protobuf/TextFormatEscaper.java create mode 100644 java/src/main/java/com/google/protobuf/UnsafeByteStrings.java (limited to 'java/src/main') diff --git a/java/src/main/java/com/google/protobuf/AbstractMessage.java b/java/src/main/java/com/google/protobuf/AbstractMessage.java index cc89173a..9f418f2b 100644 --- a/java/src/main/java/com/google/protobuf/AbstractMessage.java +++ b/java/src/main/java/com/google/protobuf/AbstractMessage.java @@ -30,6 +30,7 @@ package com.google.protobuf; +import com.google.protobuf.Descriptors.EnumValueDescriptor; import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.Descriptors.OneofDescriptor; import com.google.protobuf.Internal.EnumLite; @@ -161,10 +162,18 @@ public abstract class AbstractMessage extends AbstractMessageLite Descriptors.Descriptor descriptor = entry.getDescriptorForType(); Descriptors.FieldDescriptor key = descriptor.findFieldByName("key"); Descriptors.FieldDescriptor value = descriptor.findFieldByName("value"); - result.put(entry.getField(key), entry.getField(value)); + Object fieldValue = entry.getField(value); + if (fieldValue instanceof EnumValueDescriptor) { + fieldValue = ((EnumValueDescriptor) fieldValue).getNumber(); + } + result.put(entry.getField(key), fieldValue); while (iterator.hasNext()) { entry = (Message) iterator.next(); - result.put(entry.getField(key), entry.getField(value)); + fieldValue = entry.getField(value); + if (fieldValue instanceof EnumValueDescriptor) { + fieldValue = ((EnumValueDescriptor) fieldValue).getNumber(); + } + result.put(entry.getField(key), fieldValue); } return result; } diff --git a/java/src/main/java/com/google/protobuf/BooleanArrayList.java b/java/src/main/java/com/google/protobuf/BooleanArrayList.java index 45492d2f..70e042f5 100644 --- a/java/src/main/java/com/google/protobuf/BooleanArrayList.java +++ b/java/src/main/java/com/google/protobuf/BooleanArrayList.java @@ -68,10 +68,17 @@ final class BooleanArrayList private int size; /** - * Constructs a new mutable {@code BooleanArrayList}. + * Constructs a new mutable {@code BooleanArrayList} with default capacity. */ BooleanArrayList() { - array = new boolean[DEFAULT_CAPACITY]; + this(DEFAULT_CAPACITY); + } + + /** + * Constructs a new mutable {@code BooleanArrayList} with the provided capacity. + */ + BooleanArrayList(int capacity) { + array = new boolean[capacity]; size = 0; } diff --git a/java/src/main/java/com/google/protobuf/BoundedByteString.java b/java/src/main/java/com/google/protobuf/BoundedByteString.java index b4c3fb1b..934c9030 100644 --- a/java/src/main/java/com/google/protobuf/BoundedByteString.java +++ b/java/src/main/java/com/google/protobuf/BoundedByteString.java @@ -33,7 +33,6 @@ package com.google.protobuf; import java.io.IOException; import java.io.InvalidObjectException; import java.io.ObjectInputStream; -import java.util.NoSuchElementException; /** * This class is used to represent the substring of a {@link ByteString} over a @@ -47,7 +46,7 @@ import java.util.NoSuchElementException; * * @author carlanton@google.com (Carl Haverl) */ -class BoundedByteString extends LiteralByteString { +final class BoundedByteString extends LiteralByteString { private final int bytesOffset; private final int bytesLength; @@ -65,16 +64,7 @@ class BoundedByteString extends LiteralByteString { */ BoundedByteString(byte[] bytes, int offset, int length) { super(bytes); - if (offset < 0) { - throw new IllegalArgumentException("Offset too small: " + offset); - } - if (length < 0) { - throw new IllegalArgumentException("Length too small: " + offset); - } - if ((long) offset + length > bytes.length) { - throw new IllegalArgumentException( - "Offset+Length too large: " + offset + "+" + length); - } + checkRange(offset, offset + length, bytes.length); this.bytesOffset = offset; this.bytesLength = length; @@ -94,14 +84,7 @@ class BoundedByteString extends LiteralByteString { public byte byteAt(int index) { // We must check the index ourselves as we cannot rely on Java array index // checking for substrings. - if (index < 0) { - throw new ArrayIndexOutOfBoundsException("Index too small: " + index); - } - if (index >= size()) { - throw new ArrayIndexOutOfBoundsException( - "Index too large: " + index + ", " + size()); - } - + checkIndex(index, size()); return bytes[bytesOffset + index]; } @@ -119,8 +102,8 @@ class BoundedByteString extends LiteralByteString { // ByteString -> byte[] @Override - protected void copyToInternal(byte[] target, int sourceOffset, - int targetOffset, int numberToCopy) { + protected void copyToInternal(byte[] target, int sourceOffset, int targetOffset, + int numberToCopy) { System.arraycopy(bytes, getOffsetIntoBytes() + sourceOffset, target, targetOffset, numberToCopy); } @@ -134,47 +117,8 @@ class BoundedByteString extends LiteralByteString { return new LiteralByteString(toByteArray()); } - private void readObject(ObjectInputStream in) throws IOException { + private void readObject(@SuppressWarnings("unused") ObjectInputStream in) throws IOException { throw new InvalidObjectException( "BoundedByteStream instances are not to be serialized directly"); } - - // ================================================================= - // ByteIterator - - @Override - public ByteIterator iterator() { - return new BoundedByteIterator(); - } - - private class BoundedByteIterator implements ByteIterator { - - private int position; - private final int limit; - - private BoundedByteIterator() { - position = getOffsetIntoBytes(); - limit = position + size(); - } - - public boolean hasNext() { - return (position < limit); - } - - public Byte next() { - // Boxing calls Byte.valueOf(byte), which does not instantiate. - return nextByte(); - } - - public byte nextByte() { - if (position >= limit) { - throw new NoSuchElementException(); - } - return bytes[position++]; - } - - public void remove() { - throw new UnsupportedOperationException(); - } - } } diff --git a/java/src/main/java/com/google/protobuf/ByteString.java b/java/src/main/java/com/google/protobuf/ByteString.java index b092bc36..68f20d51 100644 --- a/java/src/main/java/com/google/protobuf/ByteString.java +++ b/java/src/main/java/com/google/protobuf/ByteString.java @@ -83,6 +83,13 @@ public abstract class ByteString implements Iterable, Serializable { */ public static final ByteString EMPTY = new LiteralByteString(new byte[0]); + /** + * Cached hash value. Intentionally accessed via a data race, which + * is safe because of the Java Memory Model's "no out-of-thin-air values" + * guarantees for ints. A value of 0 implies that the hash has not been set. + */ + private int hash = 0; + // This constructor is here to prevent subclassing outside of this package, ByteString() {} @@ -105,7 +112,38 @@ public abstract class ByteString implements Iterable, Serializable { * * @return the iterator */ - public abstract ByteIterator iterator(); + @Override + public final ByteIterator iterator() { + return new ByteIterator() { + private int position = 0; + private final int limit = size(); + + @Override + public boolean hasNext() { + return position < limit; + } + + @Override + public Byte next() { + // Boxing calls Byte.valueOf(byte), which does not instantiate. + return nextByte(); + } + + @Override + public byte nextByte() { + try { + return byteAt(position++); + } catch (ArrayIndexOutOfBoundsException e) { + throw new NoSuchElementException(e.getMessage()); + } + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } /** * This interface extends {@code Iterator}, so that we can return an @@ -134,7 +172,7 @@ public abstract class ByteString implements Iterable, Serializable { * * @return true if this is zero bytes long */ - public boolean isEmpty() { + public final boolean isEmpty() { return size() == 0; } @@ -150,7 +188,7 @@ public abstract class ByteString implements Iterable, Serializable { * @throws IndexOutOfBoundsException if {@code beginIndex < 0} or * {@code beginIndex > size()}. */ - public ByteString substring(int beginIndex) { + public final ByteString substring(int beginIndex) { return substring(beginIndex, size()); } @@ -175,7 +213,7 @@ public abstract class ByteString implements Iterable, Serializable { * argument is a prefix of the byte sequence represented by * this string; false otherwise. */ - public boolean startsWith(ByteString prefix) { + public final boolean startsWith(ByteString prefix) { return size() >= prefix.size() && substring(0, prefix.size()).equals(prefix); } @@ -189,7 +227,7 @@ public abstract class ByteString implements Iterable, Serializable { * argument is a suffix of the byte sequence represented by * this string; false otherwise. */ - public boolean endsWith(ByteString suffix) { + public final boolean endsWith(ByteString suffix) { return size() >= suffix.size() && substring(size() - suffix.size()).equals(suffix); } @@ -309,8 +347,7 @@ public abstract class ByteString implements Iterable, Serializable { */ public static ByteString readFrom(InputStream streamToDrain) throws IOException { - return readFrom( - streamToDrain, MIN_READ_FROM_CHUNK_SIZE, MAX_READ_FROM_CHUNK_SIZE); + return readFrom(streamToDrain, MIN_READ_FROM_CHUNK_SIZE, MAX_READ_FROM_CHUNK_SIZE); } /** @@ -383,10 +420,10 @@ public abstract class ByteString implements Iterable, Serializable { if (bytesRead == 0) { return null; - } else { - // Always make a copy since InputStream could steal a reference to buf. - return ByteString.copyFrom(buf, 0, bytesRead); } + + // Always make a copy since InputStream could steal a reference to buf. + return ByteString.copyFrom(buf, 0, bytesRead); } // ================================================================= @@ -402,12 +439,10 @@ public abstract class ByteString implements Iterable, Serializable { * @param other string to concatenate * @return a new {@code ByteString} instance */ - public ByteString concat(ByteString other) { - int thisSize = size(); - int otherSize = other.size(); - if ((long) thisSize + otherSize >= Integer.MAX_VALUE) { + public final ByteString concat(ByteString other) { + if (Integer.MAX_VALUE - size() < other.size()) { throw new IllegalArgumentException("ByteString would be too long: " + - thisSize + "+" + otherSize); + size() + "+" + other.size()); } return RopeByteString.concatenate(this, other); @@ -426,29 +461,29 @@ public abstract class ByteString implements Iterable, Serializable { * @return new {@code ByteString} */ public static ByteString copyFrom(Iterable byteStrings) { - Collection collection; + // Determine the size; + final int size; if (!(byteStrings instanceof Collection)) { - collection = new ArrayList(); - for (ByteString byteString : byteStrings) { - collection.add(byteString); + int tempSize = 0; + for (Iterator iter = byteStrings.iterator(); iter.hasNext(); + iter.next(), ++tempSize) { } + size = tempSize; } else { - collection = (Collection) byteStrings; + size = ((Collection) byteStrings).size(); } - ByteString result; - if (collection.isEmpty()) { - result = EMPTY; - } else { - result = balancedConcat(collection.iterator(), collection.size()); + + if (size == 0) { + return EMPTY; } - return result; + + return balancedConcat(byteStrings.iterator(), size); } // Internal function used by copyFrom(Iterable). // Create a balanced concatenation of the next "length" elements from the // iterable. - private static ByteString balancedConcat(Iterator iterator, - int length) { + private static ByteString balancedConcat(Iterator iterator, int length) { assert length >= 1; ByteString result; if (length == 1) { @@ -486,25 +521,10 @@ public abstract class ByteString implements Iterable, Serializable { * @throws IndexOutOfBoundsException if an offset or size is negative or too * large */ - public void copyTo(byte[] target, int sourceOffset, int targetOffset, + public final void copyTo(byte[] target, int sourceOffset, int targetOffset, int numberToCopy) { - if (sourceOffset < 0) { - throw new IndexOutOfBoundsException("Source offset < 0: " + sourceOffset); - } - if (targetOffset < 0) { - throw new IndexOutOfBoundsException("Target offset < 0: " + targetOffset); - } - if (numberToCopy < 0) { - throw new IndexOutOfBoundsException("Length < 0: " + numberToCopy); - } - if (sourceOffset + numberToCopy > size()) { - throw new IndexOutOfBoundsException( - "Source end offset < 0: " + (sourceOffset + numberToCopy)); - } - if (targetOffset + numberToCopy > target.length) { - throw new IndexOutOfBoundsException( - "Target end offset < 0: " + (targetOffset + numberToCopy)); - } + checkRange(sourceOffset, sourceOffset + numberToCopy, size()); + checkRange(targetOffset, targetOffset + numberToCopy, target.length); if (numberToCopy > 0) { copyToInternal(target, sourceOffset, targetOffset, numberToCopy); } @@ -534,8 +554,8 @@ public abstract class ByteString implements Iterable, Serializable { * * @return copied bytes */ - public byte[] toByteArray() { - int size = size(); + public final byte[] toByteArray() { + final int size = size(); if (size == 0) { return Internal.EMPTY_BYTE_ARRAY; } @@ -548,6 +568,10 @@ public abstract class ByteString implements Iterable, Serializable { * Writes the complete contents of this byte string to * the specified output stream argument. * + *

It is assumed that the {@link OutputStream} will not modify the contents passed it + * it. It may be possible for a malicious {@link OutputStream} to corrupt + * the data underlying the {@link ByteString}. + * * @param out the output stream to which to write the data. * @throws IOException if an I/O error occurs. */ @@ -563,30 +587,20 @@ public abstract class ByteString implements Iterable, Serializable { * @throws IndexOutOfBoundsException if an offset or size is negative or too * large */ - void writeTo(OutputStream out, int sourceOffset, int numberToWrite) + final void writeTo(OutputStream out, int sourceOffset, int numberToWrite) throws IOException { - if (sourceOffset < 0) { - throw new IndexOutOfBoundsException("Source offset < 0: " + sourceOffset); - } - if (numberToWrite < 0) { - throw new IndexOutOfBoundsException("Length < 0: " + numberToWrite); - } - if (sourceOffset + numberToWrite > size()) { - throw new IndexOutOfBoundsException( - "Source end offset exceeded: " + (sourceOffset + numberToWrite)); - } + checkRange(sourceOffset, sourceOffset + numberToWrite, size()); if (numberToWrite > 0) { writeToInternal(out, sourceOffset, numberToWrite); } - } /** * Internal version of {@link #writeTo(OutputStream,int,int)} that assumes * all error checking has already been done. */ - abstract void writeToInternal(OutputStream out, int sourceOffset, - int numberToWrite) throws IOException; + abstract void writeToInternal(OutputStream out, int sourceOffset, int numberToWrite) + throws IOException; /** * Constructs a read-only {@code java.nio.ByteBuffer} whose content @@ -618,7 +632,7 @@ public abstract class ByteString implements Iterable, Serializable { * @return new string * @throws UnsupportedEncodingException if charset isn't recognized */ - public String toString(String charsetName) + public final String toString(String charsetName) throws UnsupportedEncodingException { try { return toString(Charset.forName(charsetName)); @@ -636,7 +650,7 @@ public abstract class ByteString implements Iterable, Serializable { * @param charset encode using this charset * @return new string */ - public String toString(Charset charset) { + public final String toString(Charset charset) { return size() == 0 ? "" : toStringInternal(charset); } @@ -657,7 +671,7 @@ public abstract class ByteString implements Iterable, Serializable { * * @return new string using UTF-8 encoding */ - public String toStringUtf8() { + public final String toStringUtf8() { return toString(Internal.UTF_8); } @@ -716,13 +730,51 @@ public abstract class ByteString implements Iterable, Serializable { public abstract boolean equals(Object o); /** - * Return a non-zero hashCode depending only on the sequence of bytes - * in this ByteString. + * Base class for leaf {@link ByteString}s (i.e. non-ropes). + */ + abstract static class LeafByteString extends ByteString { + @Override + protected final int getTreeDepth() { + return 0; + } + + @Override + protected final boolean isBalanced() { + return true; + } + + /** + * Check equality of the substring of given length of this object starting at + * zero with another {@code ByteString} substring starting at offset. + * + * @param other what to compare a substring in + * @param offset offset into other + * @param length number of bytes to compare + * @return true for equality of substrings, else false. + */ + abstract boolean equalsRange(ByteString other, int offset, int length); + } + + /** + * Compute the hashCode using the traditional algorithm from {@link + * ByteString}. * - * @return hashCode value for this object + * @return hashCode value */ @Override - public abstract int hashCode(); + public final int hashCode() { + int h = hash; + + if (h == 0) { + int size = size(); + h = partialHash(size, 0, size); + if (h == 0) { + h = 1; + } + hash = h; + } + return h; + } // ================================================================= // Input stream @@ -1034,7 +1086,9 @@ public abstract class ByteString implements Iterable, Serializable { * * @return value of cached hash code or 0 if not computed yet */ - protected abstract int peekCachedHashCode(); + protected final int peekCachedHashCode() { + return hash; + } /** * Compute the hash across the value bytes starting with the given hash, and @@ -1049,8 +1103,49 @@ public abstract class ByteString implements Iterable, Serializable { */ protected abstract int partialHash(int h, int offset, int length); + /** + * Checks that the given index falls within the specified array size. + * + * @param index the index position to be tested + * @param size the length of the array + * @throws ArrayIndexOutOfBoundsException if the index does not fall within the array. + */ + static void checkIndex(int index, int size) { + if ((index | (size - (index + 1))) < 0) { + if (index < 0) { + throw new ArrayIndexOutOfBoundsException("Index < 0: " + index); + } + throw new ArrayIndexOutOfBoundsException("Index > length: " + index + ", " + size); + } + } + + /** + * Checks that the given range falls within the bounds of an array + * + * @param startIndex the start index of the range (inclusive) + * @param endIndex the end index of the range (exclusive) + * @param size the size of the array. + * @return the length of the range. + * @throws ArrayIndexOutOfBoundsException some or all of the range falls outside of the array. + */ + static int checkRange(int startIndex, int endIndex, int size) { + final int length = endIndex - startIndex; + if ((startIndex | endIndex | length | (size - endIndex)) < 0) { + if (startIndex < 0) { + throw new IndexOutOfBoundsException("Beginning index: " + startIndex + " < 0"); + } + if (endIndex < startIndex) { + throw new IndexOutOfBoundsException( + "Beginning index larger than ending index: " + startIndex + ", " + endIndex); + } + // endIndex >= size + throw new IndexOutOfBoundsException("End index: " + endIndex + " >= " + size); + } + return length; + } + @Override - public String toString() { + public final String toString() { return String.format("", Integer.toHexString(System.identityHashCode(this)), size()); } diff --git a/java/src/main/java/com/google/protobuf/CodedInputStream.java b/java/src/main/java/com/google/protobuf/CodedInputStream.java index d201f7c5..adc91536 100644 --- a/java/src/main/java/com/google/protobuf/CodedInputStream.java +++ b/java/src/main/java/com/google/protobuf/CodedInputStream.java @@ -1055,20 +1055,6 @@ public final class CodedInputStream { private RefillCallback refillCallback = null; - /** - * Ensures that at least {@code n} bytes are available in the buffer, reading - * more bytes from the input if necessary to make it so. Caller must ensure - * that the requested space is less than BUFFER_SIZE. - * - * @throws InvalidProtocolBufferException The end of the stream or the current - * limit was reached. - */ - private void ensureAvailable(int n) throws IOException { - if (bufferSize - bufferPos < n) { - refillBuffer(n); - } - } - /** * Reads more bytes from the input, making at least {@code n} bytes available * in the buffer. Caller must ensure that the requested space is not yet @@ -1180,86 +1166,97 @@ public final class CodedInputStream { } } - if (totalBytesRetired + bufferPos + size > currentLimit) { + // Verify that the message size so far has not exceeded sizeLimit. + int currentMessageSize = totalBytesRetired + bufferPos + size; + if (currentMessageSize > sizeLimit) { + throw InvalidProtocolBufferException.sizeLimitExceeded(); + } + + // Verify that the message size so far has not exceeded currentLimit. + if (currentMessageSize > currentLimit) { // Read to the end of the stream anyway. skipRawBytes(currentLimit - totalBytesRetired - bufferPos); - // Then fail. throw InvalidProtocolBufferException.truncatedMessage(); } - if (size < BUFFER_SIZE) { - // Reading more bytes than are in the buffer, but not an excessive number - // of bytes. We can safely allocate the resulting array ahead of time. + // We need the input stream to proceed. + if (input == null) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + + final int originalBufferPos = bufferPos; + final int bufferedBytes = bufferSize - bufferPos; + + // Mark the current buffer consumed. + totalBytesRetired += bufferSize; + bufferPos = 0; + bufferSize = 0; - // First copy what we have. + // Determine the number of bytes we need to read from the input stream. + int sizeLeft = size - bufferedBytes; + // TODO(nathanmittler): Consider using a value larger than BUFFER_SIZE. + if (sizeLeft < BUFFER_SIZE || sizeLeft <= input.available()) { + // Either the bytes we need are known to be available, or the required buffer is + // within an allowed threshold - go ahead and allocate the buffer now. final byte[] bytes = new byte[size]; - int pos = bufferSize - bufferPos; - System.arraycopy(buffer, bufferPos, bytes, 0, pos); - bufferPos = bufferSize; - // We want to refill the buffer and then copy from the buffer into our - // byte array rather than reading directly into our byte array because - // the input may be unbuffered. - ensureAvailable(size - pos); - System.arraycopy(buffer, 0, bytes, pos, size - pos); - bufferPos = size - pos; + // Copy all of the buffered bytes to the result buffer. + System.arraycopy(buffer, originalBufferPos, bytes, 0, bufferedBytes); - return bytes; - } else { - // The size is very large. For security reasons, we can't allocate the - // entire byte array yet. The size comes directly from the input, so a - // maliciously-crafted message could provide a bogus very large size in - // order to trick the app into allocating a lot of memory. We avoid this - // by allocating and reading only a small chunk at a time, so that the - // malicious message must actually *be* extremely large to cause - // problems. Meanwhile, we limit the allowed size of a message elsewhere. - - // Remember the buffer markers since we'll have to copy the bytes out of - // it later. - final int originalBufferPos = bufferPos; - final int originalBufferSize = bufferSize; - - // Mark the current buffer consumed. - totalBytesRetired += bufferSize; - bufferPos = 0; - bufferSize = 0; - - // Read all the rest of the bytes we need. - int sizeLeft = size - (originalBufferSize - originalBufferPos); - final List chunks = new ArrayList(); - - while (sizeLeft > 0) { - final byte[] chunk = new byte[Math.min(sizeLeft, BUFFER_SIZE)]; - int pos = 0; - while (pos < chunk.length) { - final int n = (input == null) ? -1 : - input.read(chunk, pos, chunk.length - pos); - if (n == -1) { - throw InvalidProtocolBufferException.truncatedMessage(); - } - totalBytesRetired += n; - pos += n; + // Fill the remaining bytes from the input stream. + int pos = bufferedBytes; + while (pos < bytes.length) { + int n = input.read(bytes, pos, size - pos); + if (n == -1) { + throw InvalidProtocolBufferException.truncatedMessage(); } - sizeLeft -= chunk.length; - chunks.add(chunk); + totalBytesRetired += n; + pos += n; } - // OK, got everything. Now concatenate it all into one buffer. - final byte[] bytes = new byte[size]; - - // Start by copying the leftover bytes from this.buffer. - int pos = originalBufferSize - originalBufferPos; - System.arraycopy(buffer, originalBufferPos, bytes, 0, pos); + return bytes; + } - // And now all the chunks. - for (final byte[] chunk : chunks) { - System.arraycopy(chunk, 0, bytes, pos, chunk.length); - pos += chunk.length; + // The size is very large. For security reasons, we can't allocate the + // entire byte array yet. The size comes directly from the input, so a + // maliciously-crafted message could provide a bogus very large size in + // order to trick the app into allocating a lot of memory. We avoid this + // by allocating and reading only a small chunk at a time, so that the + // malicious message must actually *be* extremely large to cause + // problems. Meanwhile, we limit the allowed size of a message elsewhere. + final List chunks = new ArrayList(); + + while (sizeLeft > 0) { + // TODO(nathanmittler): Consider using a value larger than BUFFER_SIZE. + final byte[] chunk = new byte[Math.min(sizeLeft, BUFFER_SIZE)]; + int pos = 0; + while (pos < chunk.length) { + final int n = input.read(chunk, pos, chunk.length - pos); + if (n == -1) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + totalBytesRetired += n; + pos += n; } + sizeLeft -= chunk.length; + chunks.add(chunk); + } - // Done. - return bytes; + // OK, got everything. Now concatenate it all into one buffer. + final byte[] bytes = new byte[size]; + + // Start by copying the leftover bytes from this.buffer. + System.arraycopy(buffer, originalBufferPos, bytes, 0, bufferedBytes); + + // And now all the chunks. + int pos = bufferedBytes; + for (final byte[] chunk : chunks) { + System.arraycopy(chunk, 0, bytes, pos, chunk.length); + pos += chunk.length; } + + // Done. + return bytes; } /** diff --git a/java/src/main/java/com/google/protobuf/CodedOutputStream.java b/java/src/main/java/com/google/protobuf/CodedOutputStream.java index 291bd20a..d8ebad21 100644 --- a/java/src/main/java/com/google/protobuf/CodedOutputStream.java +++ b/java/src/main/java/com/google/protobuf/CodedOutputStream.java @@ -53,7 +53,7 @@ import java.util.logging.Logger; * @author kneton@google.com Kenton Varda */ public final class CodedOutputStream { - + private static final Logger logger = Logger.getLogger(CodedOutputStream.class.getName()); // TODO(dweis): Consider migrating to a ByteBuffer. @@ -243,19 +243,6 @@ public final class CodedOutputStream { } - /** - * Write a group represented by an {@link UnknownFieldSet}. - * - * @deprecated UnknownFieldSet now implements MessageLite, so you can just - * call {@link #writeGroup}. - */ - @Deprecated - public void writeUnknownGroup(final int fieldNumber, - final MessageLite value) - throws IOException { - writeGroup(fieldNumber, value); - } - /** Write an embedded message field, including tag, to the stream. */ public void writeMessage(final int fieldNumber, final MessageLite value) throws IOException { @@ -428,7 +415,7 @@ public final class CodedOutputStream { try { efficientWriteStringNoTag(value); } catch (UnpairedSurrogateException e) { - logger.log(Level.WARNING, + logger.log(Level.WARNING, "Converting ill-formed UTF-16. Your Protocol Buffer will not round trip correctly!", e); inefficientWriteStringNoTag(value); } @@ -449,10 +436,10 @@ public final class CodedOutputStream { * Write a {@code string} field to the stream efficiently. If the {@code string} is malformed, * this method rolls back its changes and throws an {@link UnpairedSurrogateException} with the * intent that the caller will catch and retry with {@link #inefficientWriteStringNoTag(String)}. - * + * * @param value the string to write to the stream - * - * @throws UnpairedSurrogateException when {@code value} is ill-formed UTF-16. + * + * @throws UnpairedSurrogateException when {@code value} is ill-formed UTF-16. */ private void efficientWriteStringNoTag(final String value) throws IOException { // UTF-8 byte length of the string is at least its UTF-16 code unit length (value.length()), @@ -510,18 +497,6 @@ public final class CodedOutputStream { } - /** - * Write a group represented by an {@link UnknownFieldSet}. - * - * @deprecated UnknownFieldSet now implements MessageLite, so you can just - * call {@link #writeGroupNoTag}. - */ - @Deprecated - public void writeUnknownGroupNoTag(final MessageLite value) - throws IOException { - writeGroupNoTag(value); - } - /** Write an embedded message field to the stream. */ public void writeMessageNoTag(final MessageLite value) throws IOException { writeRawVarint32(value.getSerializedSize()); @@ -684,20 +659,6 @@ public final class CodedOutputStream { return computeTagSize(fieldNumber) * 2 + computeGroupSizeNoTag(value); } - /** - * Compute the number of bytes that would be needed to encode a - * {@code group} field represented by an {@code UnknownFieldSet}, including - * tag. - * - * @deprecated UnknownFieldSet now implements MessageLite, so you can just - * call {@link #computeGroupSize}. - */ - @Deprecated - public static int computeUnknownGroupSize(final int fieldNumber, - final MessageLite value) { - return computeGroupSize(fieldNumber, value); - } - /** * Compute the number of bytes that would be needed to encode an * embedded message field, including tag. @@ -926,19 +887,6 @@ public final class CodedOutputStream { return value.getSerializedSize(); } - /** - * Compute the number of bytes that would be needed to encode a - * {@code group} field represented by an {@code UnknownFieldSet}, including - * tag. - * - * @deprecated UnknownFieldSet now implements MessageLite, so you can just - * call {@link #computeUnknownGroupSizeNoTag}. - */ - @Deprecated - public static int computeUnknownGroupSizeNoTag(final MessageLite value) { - return computeGroupSizeNoTag(value); - } - /** * Compute the number of bytes that would be needed to encode an embedded * message field. @@ -1295,10 +1243,10 @@ public final class CodedOutputStream { * negative. */ public static int computeRawVarint32Size(final int value) { - if ((value & (0xffffffff << 7)) == 0) return 1; - if ((value & (0xffffffff << 14)) == 0) return 2; - if ((value & (0xffffffff << 21)) == 0) return 3; - if ((value & (0xffffffff << 28)) == 0) return 4; + if ((value & (~0 << 7)) == 0) return 1; + if ((value & (~0 << 14)) == 0) return 2; + if ((value & (~0 << 21)) == 0) return 3; + if ((value & (~0 << 28)) == 0) return 4; return 5; } @@ -1316,17 +1264,16 @@ public final class CodedOutputStream { } /** Compute the number of bytes that would be needed to encode a varint. */ - public static int computeRawVarint64Size(final long value) { - if ((value & (0xffffffffffffffffL << 7)) == 0) return 1; - if ((value & (0xffffffffffffffffL << 14)) == 0) return 2; - if ((value & (0xffffffffffffffffL << 21)) == 0) return 3; - if ((value & (0xffffffffffffffffL << 28)) == 0) return 4; - if ((value & (0xffffffffffffffffL << 35)) == 0) return 5; - if ((value & (0xffffffffffffffffL << 42)) == 0) return 6; - if ((value & (0xffffffffffffffffL << 49)) == 0) return 7; - if ((value & (0xffffffffffffffffL << 56)) == 0) return 8; - if ((value & (0xffffffffffffffffL << 63)) == 0) return 9; - return 10; + public static int computeRawVarint64Size(long value) { + // handle two popular special cases up front ... + if ((value & (~0L << 7)) == 0L) return 1; + if (value < 0L) return 10; + // ... leaving us with 8 remaining, which we can divide and conquer + int n = 2; + if ((value & (~0L << 35)) != 0L) { n += 4; value >>>= 28; } + if ((value & (~0L << 21)) != 0L) { n += 2; value >>>= 14; } + if ((value & (~0L << 14)) != 0L) { n += 1; } + return n; } /** Write a little-endian 32-bit integer. */ diff --git a/java/src/main/java/com/google/protobuf/Descriptors.java b/java/src/main/java/com/google/protobuf/Descriptors.java index 7cfc47f7..5e15cfbe 100644 --- a/java/src/main/java/com/google/protobuf/Descriptors.java +++ b/java/src/main/java/com/google/protobuf/Descriptors.java @@ -889,6 +889,11 @@ public final class Descriptors { */ public String getFullName() { return fullName; } + /** Get the JSON name of this field. */ + public String getJsonName() { + return jsonName; + } + /** * Get the field's java type. This is just for convenience. Every * {@code FieldDescriptorProto.Type} maps to exactly one Java type. @@ -1079,6 +1084,7 @@ public final class Descriptors { private FieldDescriptorProto proto; private final String fullName; + private final String jsonName; private final FileDescriptor file; private final Descriptor extensionScope; @@ -1157,6 +1163,38 @@ public final class Descriptors { private final Object defaultDefault; } + // TODO(xiaofeng): Implement it consistently across different languages. See b/24751348. + private static String fieldNameToLowerCamelCase(String name) { + StringBuilder result = new StringBuilder(name.length()); + boolean isNextUpperCase = false; + for (int i = 0; i < name.length(); i++) { + Character ch = name.charAt(i); + if (Character.isLowerCase(ch)) { + if (isNextUpperCase) { + result.append(Character.toUpperCase(ch)); + } else { + result.append(ch); + } + isNextUpperCase = false; + } else if (Character.isUpperCase(ch)) { + if (i == 0) { + // Force first letter to lower-case. + result.append(Character.toLowerCase(ch)); + } else { + // Capital letters after the first are left as-is. + result.append(ch); + } + isNextUpperCase = false; + } else if (Character.isDigit(ch)) { + result.append(ch); + isNextUpperCase = false; + } else { + isNextUpperCase = true; + } + } + return result.toString(); + } + private FieldDescriptor(final FieldDescriptorProto proto, final FileDescriptor file, final Descriptor parent, @@ -1167,6 +1205,11 @@ public final class Descriptors { this.proto = proto; fullName = computeFullName(file, parent, proto.getName()); this.file = file; + if (proto.hasJsonName()) { + jsonName = proto.getJsonName(); + } else { + jsonName = fieldNameToLowerCamelCase(proto.getName()); + } if (proto.hasType()) { type = Type.valueOf(proto.getType()); diff --git a/java/src/main/java/com/google/protobuf/DoubleArrayList.java b/java/src/main/java/com/google/protobuf/DoubleArrayList.java index 90ebe109..bcc9d6ee 100644 --- a/java/src/main/java/com/google/protobuf/DoubleArrayList.java +++ b/java/src/main/java/com/google/protobuf/DoubleArrayList.java @@ -68,10 +68,17 @@ final class DoubleArrayList private int size; /** - * Constructs a new mutable {@code DoubleArrayList}. + * Constructs a new mutable {@code DoubleArrayList} with default capacity. */ DoubleArrayList() { - array = new double[DEFAULT_CAPACITY]; + this(DEFAULT_CAPACITY); + } + + /** + * Constructs a new mutable {@code DoubleArrayList} with the provided capacity. + */ + DoubleArrayList(int capacity) { + array = new double[capacity]; size = 0; } diff --git a/java/src/main/java/com/google/protobuf/FloatArrayList.java b/java/src/main/java/com/google/protobuf/FloatArrayList.java index 293eaff6..033b5eed 100644 --- a/java/src/main/java/com/google/protobuf/FloatArrayList.java +++ b/java/src/main/java/com/google/protobuf/FloatArrayList.java @@ -67,10 +67,17 @@ final class FloatArrayList extends AbstractProtobufList implements FloatL private int size; /** - * Constructs a new mutable {@code FloatArrayList}. + * Constructs a new mutable {@code FloatArrayList} with default capacity. */ FloatArrayList() { - array = new float[DEFAULT_CAPACITY]; + this(DEFAULT_CAPACITY); + } + + /** + * Constructs a new mutable {@code FloatArrayList} with the provided capacity. + */ + FloatArrayList(int capacity) { + array = new float[capacity]; size = 0; } diff --git a/java/src/main/java/com/google/protobuf/GeneratedMessageLite.java b/java/src/main/java/com/google/protobuf/GeneratedMessageLite.java index 4316efee..81e1862c 100644 --- a/java/src/main/java/com/google/protobuf/GeneratedMessageLite.java +++ b/java/src/main/java/com/google/protobuf/GeneratedMessageLite.java @@ -102,6 +102,11 @@ public abstract class GeneratedMessageLite< * @return {@code true} unless the tag is an end-group tag. */ protected boolean parseUnknownField(int tag, CodedInputStream input) throws IOException { + // This will avoid the allocation of unknown fields when a group tag is encountered. + if (WireFormat.getTagWireType(tag) == WireFormat.WIRETYPE_END_GROUP) { + return false; + } + ensureUnknownFieldsInitialized(); return unknownFields.mergeFieldFrom(tag, input); } @@ -1173,6 +1178,10 @@ public abstract class GeneratedMessageLite< return new IntArrayList(); } + protected static IntList newIntListWithCapacity(int capacity) { + return new IntArrayList(capacity); + } + protected static IntList newIntList(List toCopy) { return new IntArrayList(toCopy); } @@ -1180,10 +1189,14 @@ public abstract class GeneratedMessageLite< protected static IntList emptyIntList() { return IntArrayList.emptyList(); } - + protected static LongList newLongList() { return new LongArrayList(); } + + protected static LongList newLongListWithCapacity(int capacity) { + return new LongArrayList(capacity); + } protected static LongList newLongList(List toCopy) { return new LongArrayList(toCopy); @@ -1197,6 +1210,10 @@ public abstract class GeneratedMessageLite< return new FloatArrayList(); } + protected static FloatList newFloatListWithCapacity(int capacity) { + return new FloatArrayList(capacity); + } + protected static FloatList newFloatList(List toCopy) { return new FloatArrayList(toCopy); } @@ -1209,6 +1226,10 @@ public abstract class GeneratedMessageLite< return new DoubleArrayList(); } + protected static DoubleList newDoubleListWithCapacity(int capacity) { + return new DoubleArrayList(capacity); + } + protected static DoubleList newDoubleList(List toCopy) { return new DoubleArrayList(toCopy); } @@ -1221,6 +1242,10 @@ public abstract class GeneratedMessageLite< return new BooleanArrayList(); } + protected static BooleanList newBooleanListWithCapacity(int capacity) { + return new BooleanArrayList(capacity); + } + protected static BooleanList newBooleanList(List toCopy) { return new BooleanArrayList(toCopy); } @@ -1237,6 +1262,10 @@ public abstract class GeneratedMessageLite< return new ProtobufArrayList(toCopy); } + protected static ProtobufList newProtobufListWithCapacity(int capacity) { + return new ProtobufArrayList(capacity); + } + protected static ProtobufList emptyProtobufList() { return ProtobufArrayList.emptyList(); } diff --git a/java/src/main/java/com/google/protobuf/IntArrayList.java b/java/src/main/java/com/google/protobuf/IntArrayList.java index f7609cc9..f4e68ed8 100644 --- a/java/src/main/java/com/google/protobuf/IntArrayList.java +++ b/java/src/main/java/com/google/protobuf/IntArrayList.java @@ -67,10 +67,17 @@ final class IntArrayList extends AbstractProtobufList implements IntLis private int size; /** - * Constructs a new mutable {@code IntArrayList}. + * Constructs a new mutable {@code IntArrayList} with default capacity. */ IntArrayList() { - array = new int[DEFAULT_CAPACITY]; + this(DEFAULT_CAPACITY); + } + + /** + * Constructs a new mutable {@code IntArrayList} with the provided capacity. + */ + IntArrayList(int capacity) { + array = new int[capacity]; size = 0; } diff --git a/java/src/main/java/com/google/protobuf/LiteralByteString.java b/java/src/main/java/com/google/protobuf/LiteralByteString.java index c5a8512a..a18c2792 100644 --- a/java/src/main/java/com/google/protobuf/LiteralByteString.java +++ b/java/src/main/java/com/google/protobuf/LiteralByteString.java @@ -36,9 +36,8 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.charset.Charset; -import java.util.ArrayList; +import java.util.Collections; import java.util.List; -import java.util.NoSuchElementException; /** * This class implements a {@link com.google.protobuf.ByteString} backed by a @@ -49,8 +48,7 @@ import java.util.NoSuchElementException; * * @author carlanton@google.com (Carl Haverl) */ -class LiteralByteString extends ByteString { - +class LiteralByteString extends ByteString.LeafByteString { private static final long serialVersionUID = 1L; protected final byte[] bytes; @@ -82,77 +80,56 @@ class LiteralByteString extends ByteString { // ByteString -> substring @Override - public ByteString substring(int beginIndex, int endIndex) { - if (beginIndex < 0) { - throw new IndexOutOfBoundsException( - "Beginning index: " + beginIndex + " < 0"); - } - if (endIndex > size()) { - throw new IndexOutOfBoundsException("End index: " + endIndex + " > " + - size()); - } - int substringLength = endIndex - beginIndex; - if (substringLength < 0) { - throw new IndexOutOfBoundsException( - "Beginning index larger than ending index: " + beginIndex + ", " - + endIndex); - } + public final ByteString substring(int beginIndex, int endIndex) { + final int length = checkRange(beginIndex, endIndex, size()); - ByteString result; - if (substringLength == 0) { - result = ByteString.EMPTY; - } else { - result = new BoundedByteString(bytes, getOffsetIntoBytes() + beginIndex, - substringLength); + if (length == 0) { + return ByteString.EMPTY; } - return result; + + return new BoundedByteString(bytes, getOffsetIntoBytes() + beginIndex, length); } // ================================================================= // ByteString -> byte[] @Override - protected void copyToInternal(byte[] target, int sourceOffset, - int targetOffset, int numberToCopy) { + protected void copyToInternal( + byte[] target, int sourceOffset, int targetOffset, int numberToCopy) { // Optimized form, not for subclasses, since we don't call // getOffsetIntoBytes() or check the 'numberToCopy' parameter. + // TODO(nathanmittler): Is not calling getOffsetIntoBytes really saving that much? System.arraycopy(bytes, sourceOffset, target, targetOffset, numberToCopy); } @Override - public void copyTo(ByteBuffer target) { - target.put(bytes, getOffsetIntoBytes(), size()); // Copies bytes + public final void copyTo(ByteBuffer target) { + target.put(bytes, getOffsetIntoBytes(), size()); // Copies bytes + } + + @Override + public final ByteBuffer asReadOnlyByteBuffer() { + return ByteBuffer.wrap(bytes, getOffsetIntoBytes(), size()).asReadOnlyBuffer(); } @Override - public ByteBuffer asReadOnlyByteBuffer() { - ByteBuffer byteBuffer = - ByteBuffer.wrap(bytes, getOffsetIntoBytes(), size()); - return byteBuffer.asReadOnlyBuffer(); + public final List asReadOnlyByteBufferList() { + return Collections.singletonList(asReadOnlyByteBuffer()); } @Override - public List asReadOnlyByteBufferList() { - // Return the ByteBuffer generated by asReadOnlyByteBuffer() as a singleton - List result = new ArrayList(1); - result.add(asReadOnlyByteBuffer()); - return result; - } - - @Override - public void writeTo(OutputStream outputStream) throws IOException { + public final void writeTo(OutputStream outputStream) throws IOException { outputStream.write(toByteArray()); } @Override - void writeToInternal(OutputStream outputStream, int sourceOffset, - int numberToWrite) throws IOException { - outputStream.write(bytes, getOffsetIntoBytes() + sourceOffset, - numberToWrite); + final void writeToInternal(OutputStream outputStream, int sourceOffset, int numberToWrite) + throws IOException { + outputStream.write(bytes, getOffsetIntoBytes() + sourceOffset, numberToWrite); } @Override - protected String toStringInternal(Charset charset) { + protected final String toStringInternal(Charset charset) { return new String(bytes, getOffsetIntoBytes(), size(), charset); } @@ -160,13 +137,13 @@ class LiteralByteString extends ByteString { // UTF-8 decoding @Override - public boolean isValidUtf8() { + public final boolean isValidUtf8() { int offset = getOffsetIntoBytes(); return Utf8.isValidUtf8(bytes, offset, offset + size()); } @Override - protected int partialIsValidUtf8(int state, int offset, int length) { + protected final int partialIsValidUtf8(int state, int offset, int length) { int index = getOffsetIntoBytes() + offset; return Utf8.partialIsValidUtf8(state, bytes, index, index + length); } @@ -175,7 +152,7 @@ class LiteralByteString extends ByteString { // equals() and hashCode() @Override - public boolean equals(Object other) { + public final boolean equals(Object other) { if (other == this) { return true; } @@ -194,19 +171,16 @@ class LiteralByteString extends ByteString { LiteralByteString otherAsLiteral = (LiteralByteString) other; // If we know the hash codes and they are not equal, we know the byte // strings are not equal. - if (hash != 0 - && otherAsLiteral.hash != 0 - && hash != otherAsLiteral.hash) { + int thisHash = peekCachedHashCode(); + int thatHash = otherAsLiteral.peekCachedHashCode(); + if (thisHash != 0 && thatHash != 0 && thisHash != thatHash) { return false; } return equalsRange((LiteralByteString) other, 0, size()); - } else if (other instanceof RopeByteString) { - return other.equals(this); } else { - throw new IllegalArgumentException( - "Has a new type of ByteString been created? Found " - + other.getClass()); + // RopeByteString and NioByteString. + return other.equals(this); } } @@ -219,65 +193,36 @@ class LiteralByteString extends ByteString { * @param length number of bytes to compare * @return true for equality of substrings, else false. */ - boolean equalsRange(LiteralByteString other, int offset, int length) { + @Override + final boolean equalsRange(ByteString other, int offset, int length) { if (length > other.size()) { - throw new IllegalArgumentException( - "Length too large: " + length + size()); + throw new IllegalArgumentException("Length too large: " + length + size()); } if (offset + length > other.size()) { throw new IllegalArgumentException( - "Ran off end of other: " + offset + ", " + length + ", " + - other.size()); + "Ran off end of other: " + offset + ", " + length + ", " + other.size()); } - byte[] thisBytes = bytes; - byte[] otherBytes = other.bytes; - int thisLimit = getOffsetIntoBytes() + length; - for (int thisIndex = getOffsetIntoBytes(), otherIndex = - other.getOffsetIntoBytes() + offset; - (thisIndex < thisLimit); ++thisIndex, ++otherIndex) { - if (thisBytes[thisIndex] != otherBytes[otherIndex]) { - return false; - } - } - return true; - } - - /** - * Cached hash value. Intentionally accessed via a data race, which - * is safe because of the Java Memory Model's "no out-of-thin-air values" - * guarantees for ints. - */ - private int hash = 0; - - /** - * Compute the hashCode using the traditional algorithm from {@link - * ByteString}. - * - * @return hashCode value - */ - @Override - public int hashCode() { - int h = hash; - - if (h == 0) { - int size = size(); - h = partialHash(size, 0, size); - if (h == 0) { - h = 1; + if (other instanceof LiteralByteString) { + LiteralByteString lbsOther = (LiteralByteString) other; + byte[] thisBytes = bytes; + byte[] otherBytes = lbsOther.bytes; + int thisLimit = getOffsetIntoBytes() + length; + for ( + int thisIndex = getOffsetIntoBytes(), otherIndex = lbsOther.getOffsetIntoBytes() + offset; + (thisIndex < thisLimit); ++thisIndex, ++otherIndex) { + if (thisBytes[thisIndex] != otherBytes[otherIndex]) { + return false; + } } - hash = h; + return true; } - return h; - } - @Override - protected int peekCachedHashCode() { - return hash; + return other.substring(offset, offset + length).equals(substring(0, length)); } @Override - protected int partialHash(int h, int offset, int length) { + protected final int partialHash(int h, int offset, int length) { return hashCode(h, bytes, getOffsetIntoBytes() + offset, length); } @@ -297,70 +242,20 @@ class LiteralByteString extends ByteString { // Input stream @Override - public InputStream newInput() { - return new ByteArrayInputStream(bytes, getOffsetIntoBytes(), - size()); // No copy + public final InputStream newInput() { + return new ByteArrayInputStream(bytes, getOffsetIntoBytes(), size()); // No copy } @Override - public CodedInputStream newCodedInput() { + public final CodedInputStream newCodedInput() { // We trust CodedInputStream not to modify the bytes, or to give anyone // else access to them. return CodedInputStream.newInstance(this); } - // ================================================================= - // ByteIterator - - @Override - public ByteIterator iterator() { - return new LiteralByteIterator(); - } - - private class LiteralByteIterator implements ByteIterator { - private int position; - private final int limit; - - private LiteralByteIterator() { - position = 0; - limit = size(); - } - - public boolean hasNext() { - return (position < limit); - } - - public Byte next() { - // Boxing calls Byte.valueOf(byte), which does not instantiate. - return nextByte(); - } - - public byte nextByte() { - try { - return bytes[position++]; - } catch (ArrayIndexOutOfBoundsException e) { - throw new NoSuchElementException(e.getMessage()); - } - } - - public void remove() { - throw new UnsupportedOperationException(); - } - } - // ================================================================= // Internal methods - @Override - protected int getTreeDepth() { - return 0; - } - - @Override - protected boolean isBalanced() { - return true; - } - /** * Offset into {@code bytes[]} to use, non-zero for substrings. * diff --git a/java/src/main/java/com/google/protobuf/LongArrayList.java b/java/src/main/java/com/google/protobuf/LongArrayList.java index 298617ff..ebe62029 100644 --- a/java/src/main/java/com/google/protobuf/LongArrayList.java +++ b/java/src/main/java/com/google/protobuf/LongArrayList.java @@ -67,10 +67,17 @@ final class LongArrayList extends AbstractProtobufList implements LongList private int size; /** - * Constructs a new mutable {@code LongArrayList}. + * Constructs a new mutable {@code LongArrayList} with default capacity. */ LongArrayList() { - array = new long[DEFAULT_CAPACITY]; + this(DEFAULT_CAPACITY); + } + + /** + * Constructs a new mutable {@code LongArrayList} with the provided capacity. + */ + LongArrayList(int capacity) { + array = new long[capacity]; size = 0; } diff --git a/java/src/main/java/com/google/protobuf/MapFieldLite.java b/java/src/main/java/com/google/protobuf/MapFieldLite.java index c17fa7b1..16d3e6d2 100644 --- a/java/src/main/java/com/google/protobuf/MapFieldLite.java +++ b/java/src/main/java/com/google/protobuf/MapFieldLite.java @@ -30,6 +30,8 @@ package com.google.protobuf; +import com.google.protobuf.Internal.EnumLite; + import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -44,7 +46,7 @@ import java.util.Set; * This class is a protobuf implementation detail. Users shouldn't use this * class directly. */ -public class MapFieldLite implements MutabilityOracle { +public final class MapFieldLite implements MutabilityOracle { private MutatabilityAwareMap mapData; private boolean isMutable; @@ -136,8 +138,9 @@ public class MapFieldLite implements MutabilityOracle { if (a instanceof byte[]) { return LiteralByteString.hashCode((byte[]) a); } - if (a instanceof Internal.EnumLite) { - return Internal.hashEnum((Internal.EnumLite) a); + // Enums should be stored as integers internally. + if (a instanceof EnumLite) { + throw new UnsupportedOperationException(); } return a.hashCode(); } diff --git a/java/src/main/java/com/google/protobuf/MessageLiteToString.java b/java/src/main/java/com/google/protobuf/MessageLiteToString.java new file mode 100644 index 00000000..e69de29b diff --git a/java/src/main/java/com/google/protobuf/NioByteString.java b/java/src/main/java/com/google/protobuf/NioByteString.java new file mode 100644 index 00000000..f71e41b2 --- /dev/null +++ b/java/src/main/java/com/google/protobuf/NioByteString.java @@ -0,0 +1,309 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InvalidObjectException; +import java.io.ObjectInputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.InvalidMarkException; +import java.nio.channels.Channels; +import java.nio.charset.Charset; +import java.util.Collections; +import java.util.List; + +/** + * A {@link ByteString} that wraps around a {@link ByteBuffer}. + */ +final class NioByteString extends ByteString.LeafByteString { + private final ByteBuffer buffer; + + NioByteString(ByteBuffer buffer) { + if (buffer == null) { + throw new NullPointerException("buffer"); + } + + this.buffer = buffer.slice(); + } + + // ================================================================= + // Serializable + + /** + * Magic method that lets us override serialization behavior. + */ + private Object writeReplace() { + return ByteString.copyFrom(buffer.slice()); + } + + /** + * Magic method that lets us override deserialization behavior. + */ + private void readObject(@SuppressWarnings("unused") ObjectInputStream in) throws IOException { + throw new InvalidObjectException("NioByteString instances are not to be serialized directly"); + } + + // ================================================================= + + @Override + public byte byteAt(int index) { + try { + return buffer.get(index); + } catch (ArrayIndexOutOfBoundsException e) { + throw e; + } catch (IndexOutOfBoundsException e) { + throw new ArrayIndexOutOfBoundsException(e.getMessage()); + } + } + + @Override + public int size() { + return buffer.remaining(); + } + + @Override + public ByteString substring(int beginIndex, int endIndex) { + try { + ByteBuffer slice = slice(beginIndex, endIndex); + return new NioByteString(slice); + } catch (ArrayIndexOutOfBoundsException e) { + throw e; + } catch (IndexOutOfBoundsException e) { + throw new ArrayIndexOutOfBoundsException(e.getMessage()); + } + } + + @Override + protected void copyToInternal( + byte[] target, int sourceOffset, int targetOffset, int numberToCopy) { + ByteBuffer slice = buffer.slice(); + slice.position(sourceOffset); + slice.get(target, targetOffset, numberToCopy); + } + + @Override + public void copyTo(ByteBuffer target) { + target.put(buffer.slice()); + } + + @Override + public void writeTo(OutputStream out) throws IOException { + writeToInternal(out, buffer.position(), buffer.remaining()); + } + + @Override + boolean equalsRange(ByteString other, int offset, int length) { + return substring(0, length).equals(other.substring(offset, offset + length)); + } + + @Override + void writeToInternal(OutputStream out, int sourceOffset, int numberToWrite) throws IOException { + if (buffer.hasArray()) { + // Optimized write for array-backed buffers. + // Note that we're taking the risk that a malicious OutputStream could modify the array. + int bufferOffset = buffer.arrayOffset() + buffer.position() + sourceOffset; + out.write(buffer.array(), bufferOffset, numberToWrite); + return; + } + + // Slow path + if (out instanceof FileOutputStream || numberToWrite >= 8192) { + // Use a channel to write out the ByteBuffer. + Channels.newChannel(out).write(slice(sourceOffset, sourceOffset + numberToWrite)); + } else { + // Just copy the data to an array and write it. + out.write(toByteArray()); + } + } + + @Override + public ByteBuffer asReadOnlyByteBuffer() { + return buffer.asReadOnlyBuffer(); + } + + @Override + public List asReadOnlyByteBufferList() { + return Collections.singletonList(asReadOnlyByteBuffer()); + } + + @Override + protected String toStringInternal(Charset charset) { + byte[] bytes; + int offset; + if (buffer.hasArray()) { + bytes = buffer.array(); + offset = buffer.arrayOffset() + buffer.position(); + } else { + bytes = toByteArray(); + offset = 0; + } + return new String(bytes, offset, size(), charset); + } + + @Override + public boolean isValidUtf8() { + // TODO(nathanmittler): add a ByteBuffer fork for Utf8.isValidUtf8 to avoid the copy + byte[] bytes; + int startIndex; + if (buffer.hasArray()) { + bytes = buffer.array(); + startIndex = buffer.arrayOffset() + buffer.position(); + } else { + bytes = toByteArray(); + startIndex = 0; + } + return Utf8.isValidUtf8(bytes, startIndex, startIndex + size()); + } + + @Override + protected int partialIsValidUtf8(int state, int offset, int length) { + // TODO(nathanmittler): TODO add a ByteBuffer fork for Utf8.partialIsValidUtf8 to avoid the copy + byte[] bytes; + int startIndex; + if (buffer.hasArray()) { + bytes = buffer.array(); + startIndex = buffer.arrayOffset() + buffer.position(); + } else { + bytes = toByteArray(); + startIndex = 0; + } + return Utf8.partialIsValidUtf8(state, bytes, startIndex, startIndex + size()); + } + + @Override + public boolean equals(Object other) { + if (other == this) { + return true; + } + if (!(other instanceof ByteString)) { + return false; + } + ByteString otherString = ((ByteString) other); + if (size() != otherString.size()) { + return false; + } + if (size() == 0) { + return true; + } + if (other instanceof NioByteString) { + return buffer.equals(((NioByteString) other).buffer); + } + if (other instanceof RopeByteString) { + return other.equals(this); + } + return buffer.equals(otherString.asReadOnlyByteBuffer()); + } + + @Override + protected int partialHash(int h, int offset, int length) { + for (int i = offset; i < offset + length; i++) { + h = h * 31 + buffer.get(i); + } + return h; + } + + @Override + public InputStream newInput() { + return new InputStream() { + private final ByteBuffer buf = buffer.slice(); + + @Override + public void mark(int readlimit) { + buf.mark(); + } + + @Override + public boolean markSupported() { + return true; + } + + @Override + public void reset() throws IOException { + try { + buf.reset(); + } catch (InvalidMarkException e) { + throw new IOException(e); + } + } + + @Override + public int available() throws IOException { + return buf.remaining(); + } + + @Override + public int read() throws IOException { + if (!buf.hasRemaining()) { + return -1; + } + return buf.get() & 0xFF; + } + + @Override + public int read(byte[] bytes, int off, int len) throws IOException { + if (!buf.hasRemaining()) { + return -1; + } + + len = Math.min(len, buf.remaining()); + buf.get(bytes, off, len); + return len; + } + }; + } + + @Override + public CodedInputStream newCodedInput() { + return CodedInputStream.newInstance(buffer); + } + + /** + * Creates a slice of a range of this buffer. + * + * @param beginIndex the beginning index of the slice (inclusive). + * @param endIndex the end index of the slice (exclusive). + * @return the requested slice. + */ + private ByteBuffer slice(int beginIndex, int endIndex) { + if (beginIndex < buffer.position() || endIndex > buffer.limit() || beginIndex > endIndex) { + throw new IllegalArgumentException( + String.format("Invalid indices [%d, %d]", beginIndex, endIndex)); + } + + ByteBuffer slice = buffer.slice(); + slice.position(beginIndex - buffer.position()); + slice.limit(endIndex - buffer.position()); + return slice; + } +} diff --git a/java/src/main/java/com/google/protobuf/ProtobufArrayList.java b/java/src/main/java/com/google/protobuf/ProtobufArrayList.java index 759368c9..d2f82ac5 100644 --- a/java/src/main/java/com/google/protobuf/ProtobufArrayList.java +++ b/java/src/main/java/com/google/protobuf/ProtobufArrayList.java @@ -60,6 +60,10 @@ class ProtobufArrayList extends AbstractProtobufList { list = new ArrayList(toCopy); } + ProtobufArrayList(int capacity) { + list = new ArrayList(capacity); + } + @Override public void add(int index, E element) { ensureIsMutable(); diff --git a/java/src/main/java/com/google/protobuf/RopeByteString.java b/java/src/main/java/com/google/protobuf/RopeByteString.java index 2c332624..6e8eb724 100644 --- a/java/src/main/java/com/google/protobuf/RopeByteString.java +++ b/java/src/main/java/com/google/protobuf/RopeByteString.java @@ -69,7 +69,7 @@ import java.util.Stack; * * @author carlanton@google.com (Carl Haverl) */ -class RopeByteString extends ByteString { +final class RopeByteString extends ByteString { /** * BAP95. Let Fn be the nth Fibonacci number. A {@link RopeByteString} of @@ -151,21 +151,24 @@ class RopeByteString extends ByteString { * @return concatenation representing the same sequence as the given strings */ static ByteString concatenate(ByteString left, ByteString right) { - ByteString result; - RopeByteString leftRope = - (left instanceof RopeByteString) ? (RopeByteString) left : null; if (right.size() == 0) { - result = left; - } else if (left.size() == 0) { - result = right; - } else { - int newLength = left.size() + right.size(); - if (newLength < ByteString.CONCATENATE_BY_COPY_SIZE) { - // Optimization from BAP95: For short (leaves in paper, but just short - // here) total length, do a copy of data to a new leaf. - result = concatenateBytes(left, right); - } else if (leftRope != null - && leftRope.right.size() + right.size() < CONCATENATE_BY_COPY_SIZE) { + return left; + } + + if (left.size() == 0) { + return right; + } + + final int newLength = left.size() + right.size(); + if (newLength < ByteString.CONCATENATE_BY_COPY_SIZE) { + // Optimization from BAP95: For short (leaves in paper, but just short + // here) total length, do a copy of data to a new leaf. + return concatenateBytes(left, right); + } + + if (left instanceof RopeByteString) { + final RopeByteString leftRope = (RopeByteString) left; + if (leftRope.right.size() + right.size() < CONCATENATE_BY_COPY_SIZE) { // Optimization from BAP95: As an optimization of the case where the // ByteString is constructed by repeated concatenate, recognize the case // where a short string is concatenated to a left-hand node whose @@ -177,9 +180,10 @@ class RopeByteString extends ByteString { // new parent node so that the depth of the result is the same as the // given left tree. ByteString newRight = concatenateBytes(leftRope.right, right); - result = new RopeByteString(leftRope.left, newRight); - } else if (leftRope != null - && leftRope.left.getTreeDepth() > leftRope.right.getTreeDepth() + return new RopeByteString(leftRope.left, newRight); + } + + if (leftRope.left.getTreeDepth() > leftRope.right.getTreeDepth() && leftRope.getTreeDepth() > right.getTreeDepth()) { // Typically for concatenate-built strings the left-side is deeper than // the right. This is our final attempt to concatenate without @@ -187,20 +191,18 @@ class RopeByteString extends ByteString { // is yet another optimization for building the string by repeatedly // concatenating on the right. ByteString newRight = new RopeByteString(leftRope.right, right); - result = new RopeByteString(leftRope.left, newRight); - } else { - // Fine, we'll add a node and increase the tree depth--unless we - // rebalance ;^) - int newDepth = Math.max(left.getTreeDepth(), right.getTreeDepth()) + 1; - if (newLength >= minLengthByDepth[newDepth]) { - // The tree is shallow enough, so don't rebalance - result = new RopeByteString(left, right); - } else { - result = new Balancer().balance(left, right); - } + return new RopeByteString(leftRope.left, newRight); } } - return result; + + // Fine, we'll add a node and increase the tree depth--unless we rebalance ;^) + int newDepth = Math.max(left.getTreeDepth(), right.getTreeDepth()) + 1; + if (newLength >= minLengthByDepth[newDepth]) { + // The tree is shallow enough, so don't rebalance + return new RopeByteString(left, right); + } + + return new Balancer().balance(left, right); } /** @@ -248,22 +250,14 @@ class RopeByteString extends ByteString { */ @Override public byte byteAt(int index) { - if (index < 0) { - throw new ArrayIndexOutOfBoundsException("Index < 0: " + index); - } - if (index > totalLength) { - throw new ArrayIndexOutOfBoundsException( - "Index > length: " + index + ", " + totalLength); - } + checkIndex(index, totalLength); - byte result; // Find the relevant piece by recursive descent if (index < leftLength) { - result = left.byteAt(index); - } else { - result = right.byteAt(index - leftLength); + return left.byteAt(index); } - return result; + + return right.byteAt(index - leftLength); } @Override @@ -309,48 +303,36 @@ class RopeByteString extends ByteString { */ @Override public ByteString substring(int beginIndex, int endIndex) { - if (beginIndex < 0) { - throw new IndexOutOfBoundsException( - "Beginning index: " + beginIndex + " < 0"); + final int length = checkRange(beginIndex, endIndex, totalLength); + + if (length == 0) { + // Empty substring + return ByteString.EMPTY; } - if (endIndex > totalLength) { - throw new IndexOutOfBoundsException( - "End index: " + endIndex + " > " + totalLength); + + if (length == totalLength) { + // The whole string + return this; } - int substringLength = endIndex - beginIndex; - if (substringLength < 0) { - throw new IndexOutOfBoundsException( - "Beginning index larger than ending index: " + beginIndex + ", " - + endIndex); + + // Proper substring + if (endIndex <= leftLength) { + // Substring on the left + return left.substring(beginIndex, endIndex); } - ByteString result; - if (substringLength == 0) { - // Empty substring - result = ByteString.EMPTY; - } else if (substringLength == totalLength) { - // The whole string - result = this; - } else { - // Proper substring - if (endIndex <= leftLength) { - // Substring on the left - result = left.substring(beginIndex, endIndex); - } else if (beginIndex >= leftLength) { - // Substring on the right - result = right - .substring(beginIndex - leftLength, endIndex - leftLength); - } else { - // Split substring - ByteString leftSub = left.substring(beginIndex); - ByteString rightSub = right.substring(0, endIndex - leftLength); - // Intentionally not rebalancing, since in many cases these two - // substrings will already be less deep than the top-level - // RopeByteString we're taking a substring of. - result = new RopeByteString(leftSub, rightSub); - } + if (beginIndex >= leftLength) { + // Substring on the right + return right.substring(beginIndex - leftLength, endIndex - leftLength); } - return result; + + // Split substring + ByteString leftSub = left.substring(beginIndex); + ByteString rightSub = right.substring(0, endIndex - leftLength); + // Intentionally not rebalancing, since in many cases these two + // substrings will already be less deep than the top-level + // RopeByteString we're taking a substring of. + return new RopeByteString(leftSub, rightSub); } // ================================================================= @@ -391,7 +373,7 @@ class RopeByteString extends ByteString { List result = new ArrayList(); PieceIterator pieces = new PieceIterator(this); while (pieces.hasNext()) { - LiteralByteString byteString = pieces.next(); + LeafByteString byteString = pieces.next(); result.add(byteString.asReadOnlyByteBuffer()); } return result; @@ -471,11 +453,10 @@ class RopeByteString extends ByteString { // hashCode if it's already computed. It's arguable we should compute the // hashCode here, and if we're going to be testing a bunch of byteStrings, // it might even make sense. - if (hash != 0) { - int cachedOtherHash = otherByteString.peekCachedHashCode(); - if (cachedOtherHash != 0 && hash != cachedOtherHash) { - return false; - } + int thisHash = peekCachedHashCode(); + int thatHash = otherByteString.peekCachedHashCode(); + if (thisHash != 0 && thatHash != 0 && thisHash != thatHash) { + return false; } return equalsFragments(otherByteString); @@ -492,12 +473,12 @@ class RopeByteString extends ByteString { */ private boolean equalsFragments(ByteString other) { int thisOffset = 0; - Iterator thisIter = new PieceIterator(this); - LiteralByteString thisString = thisIter.next(); + Iterator thisIter = new PieceIterator(this); + LeafByteString thisString = thisIter.next(); int thatOffset = 0; - Iterator thatIter = new PieceIterator(other); - LiteralByteString thatString = thatIter.next(); + Iterator thatIter = new PieceIterator(other); + LeafByteString thatString = thatIter.next(); int pos = 0; while (true) { @@ -536,33 +517,6 @@ class RopeByteString extends ByteString { } } - /** - * Cached hash value. Intentionally accessed via a data race, which is safe - * because of the Java Memory Model's "no out-of-thin-air values" guarantees - * for ints. - */ - private int hash = 0; - - @Override - public int hashCode() { - int h = hash; - - if (h == 0) { - h = totalLength; - h = partialHash(h, 0, totalLength); - if (h == 0) { - h = 1; - } - hash = h; - } - return h; - } - - @Override - protected int peekCachedHashCode() { - return hash; - } - @Override protected int partialHash(int h, int offset, int length) { int toIndex = offset + length; @@ -714,34 +668,34 @@ class RopeByteString extends ByteString { *

This iterator is used to implement * {@link RopeByteString#equalsFragments(ByteString)}. */ - private static class PieceIterator implements Iterator { + private static class PieceIterator implements Iterator { private final Stack breadCrumbs = new Stack(); - private LiteralByteString next; + private LeafByteString next; private PieceIterator(ByteString root) { next = getLeafByLeft(root); } - private LiteralByteString getLeafByLeft(ByteString root) { + private LeafByteString getLeafByLeft(ByteString root) { ByteString pos = root; while (pos instanceof RopeByteString) { RopeByteString rbs = (RopeByteString) pos; breadCrumbs.push(rbs); pos = rbs.left; } - return (LiteralByteString) pos; + return (LeafByteString) pos; } - private LiteralByteString getNextNonEmptyLeaf() { + private LeafByteString getNextNonEmptyLeaf() { while (true) { // Almost always, we go through this loop exactly once. However, if // we discover an empty string in the rope, we toss it and try again. if (breadCrumbs.isEmpty()) { return null; } else { - LiteralByteString result = getLeafByLeft(breadCrumbs.pop().right); + LeafByteString result = getLeafByLeft(breadCrumbs.pop().right); if (!result.isEmpty()) { return result; } @@ -749,6 +703,7 @@ class RopeByteString extends ByteString { } } + @Override public boolean hasNext() { return next != null; } @@ -758,15 +713,17 @@ class RopeByteString extends ByteString { * * @return next non-empty LiteralByteString or {@code null} */ - public LiteralByteString next() { + @Override + public LeafByteString next() { if (next == null) { throw new NoSuchElementException(); } - LiteralByteString result = next; + LeafByteString result = next; next = getNextNonEmptyLeaf(); return result; } + @Override public void remove() { throw new UnsupportedOperationException(); } @@ -781,52 +738,11 @@ class RopeByteString extends ByteString { return new LiteralByteString(toByteArray()); } - private void readObject(ObjectInputStream in) throws IOException { + private void readObject(@SuppressWarnings("unused") ObjectInputStream in) throws IOException { throw new InvalidObjectException( "RopeByteStream instances are not to be serialized directly"); } - // ================================================================= - // ByteIterator - - @Override - public ByteIterator iterator() { - return new RopeByteIterator(); - } - - private class RopeByteIterator implements ByteString.ByteIterator { - - private final PieceIterator pieces; - private ByteIterator bytes; - int bytesRemaining; - - private RopeByteIterator() { - pieces = new PieceIterator(RopeByteString.this); - bytes = pieces.next().iterator(); - bytesRemaining = size(); - } - - public boolean hasNext() { - return (bytesRemaining > 0); - } - - public Byte next() { - return nextByte(); // Does not instantiate a Byte - } - - public byte nextByte() { - if (!bytes.hasNext()) { - bytes = pieces.next().iterator(); - } - --bytesRemaining; - return bytes.nextByte(); - } - - public void remove() { - throw new UnsupportedOperationException(); - } - } - /** * This class is the {@link RopeByteString} equivalent for * {@link ByteArrayInputStream}. @@ -835,7 +751,7 @@ class RopeByteString extends ByteString { // Iterates through the pieces of the rope private PieceIterator pieceIterator; // The current piece - private LiteralByteString currentPiece; + private LeafByteString currentPiece; // The size of the current piece private int currentPieceSize; // The index of the next byte to read in the current piece @@ -872,7 +788,7 @@ class RopeByteString extends ByteString { /** * Internal implementation of read and skip. If b != null, then read the * next {@code length} bytes into the buffer {@code b} at - * offset {@code offset}. If b == null, then skip the next {@code length) + * offset {@code offset}. If b == null, then skip the next {@code length} * bytes. *

* This method assumes that all error checking has already happened. diff --git a/java/src/main/java/com/google/protobuf/TextFormat.java b/java/src/main/java/com/google/protobuf/TextFormat.java index 44d036c1..c99b5285 100644 --- a/java/src/main/java/com/google/protobuf/TextFormat.java +++ b/java/src/main/java/com/google/protobuf/TextFormat.java @@ -1074,6 +1074,18 @@ public final class TextFormat { private ParseException floatParseException(final NumberFormatException e) { return parseException("Couldn't parse number: " + e.getMessage()); } + + /** + * Returns a {@link UnknownFieldParseException} with the line and column + * numbers of the previous token in the description, and the unknown field + * name, suitable for throwing. + */ + public UnknownFieldParseException unknownFieldParseExceptionPreviousToken( + final String unknownField, final String description) { + // Note: People generally prefer one-based line and column numbers. + return new UnknownFieldParseException( + previousLine + 1, previousColumn + 1, unknownField, description); + } } /** Thrown when parsing an invalid text format message. */ @@ -1121,6 +1133,45 @@ public final class TextFormat { return column; } } + + /** + * Thrown when encountering an unknown field while parsing + * a text format message. + */ + public static class UnknownFieldParseException extends ParseException { + private final String unknownField; + + /** + * Create a new instance, with -1 as the line and column numbers, and an + * empty unknown field name. + */ + public UnknownFieldParseException(final String message) { + this(-1, -1, "", message); + } + + /** + * Create a new instance + * + * @param line the line number where the parse error occurred, + * using 1-offset. + * @param column the column number where the parser error occurred, + * using 1-offset. + * @param unknownField the name of the unknown field found while parsing. + */ + public UnknownFieldParseException(final int line, final int column, + final String unknownField, final String message) { + super(line, column, message); + this.unknownField = unknownField; + } + + /** + * Return the name of the unknown field encountered while parsing the + * protocol buffer string. + */ + public String getUnknownField() { + return unknownField; + } + } private static final Parser PARSER = Parser.newBuilder().build(); @@ -1388,7 +1439,8 @@ public final class TextFormat { if (field == null) { if (!allowUnknownFields) { - throw tokenizer.parseExceptionPreviousToken( + throw tokenizer.unknownFieldParseExceptionPreviousToken( + name, "Message type \"" + type.getFullName() + "\" has no field named \"" + name + "\"."); } else { diff --git a/java/src/main/java/com/google/protobuf/TextFormatEscaper.java b/java/src/main/java/com/google/protobuf/TextFormatEscaper.java new file mode 100644 index 00000000..e69de29b diff --git a/java/src/main/java/com/google/protobuf/UnsafeByteStrings.java b/java/src/main/java/com/google/protobuf/UnsafeByteStrings.java new file mode 100644 index 00000000..c1997515 --- /dev/null +++ b/java/src/main/java/com/google/protobuf/UnsafeByteStrings.java @@ -0,0 +1,55 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import java.nio.ByteBuffer; + +/** + * Provides unsafe factory methods for {@link ByteString} instances. + * + *

DISCLAIMER: The methods in this class should only be called if it is + * guaranteed that the the buffer backing the {@link ByteString} will never change! Mutation of a + * {@link ByteString} can lead to unexpected and undesirable consequences in your application, + * and will likely be difficult to debug. Proceed with caution! + */ +public final class UnsafeByteStrings { + private UnsafeByteStrings() {} + + /** + * An unsafe operation that returns a {@link ByteString} that is backed by the provided buffer. + * + * @param buffer the Java NIO buffer to be wrapped. + * @return a {@link ByteString} backed by the provided buffer. + */ + public static ByteString unsafeWrap(ByteBuffer buffer) { + return new NioByteString(buffer); + } +} -- cgit v1.2.3