diff options
author | Mitsuru Oshima <oshima@google.com> | 2009-06-10 00:30:10 -0700 |
---|---|---|
committer | Mitsuru Oshima <oshima@google.com> | 2009-06-10 10:28:53 -0700 |
commit | babfb77851cabbf713f91dcee2836f46628dcff4 (patch) | |
tree | 9fd4876246f0dd05876c2323e79cbff497c73ab3 /src/com/google/common/io/protocol/ProtoBuf.java | |
parent | 5a08423c59281b29b87fab0f5b53a81e8d3e0da7 (diff) | |
download | protobuf-babfb77851cabbf713f91dcee2836f46628dcff4.tar.gz protobuf-babfb77851cabbf713f91dcee2836f46628dcff4.tar.bz2 protobuf-babfb77851cabbf713f91dcee2836f46628dcff4.zip |
ProtoBuf update
* Added IntMap that uses Hashtable for larger keys
* Chagned to use IntMap to allow larger tags
* Changed to use autoboxing for int/longs.
Diffstat (limited to 'src/com/google/common/io/protocol/ProtoBuf.java')
-rw-r--r-- | src/com/google/common/io/protocol/ProtoBuf.java | 316 |
1 files changed, 188 insertions, 128 deletions
diff --git a/src/com/google/common/io/protocol/ProtoBuf.java b/src/com/google/common/io/protocol/ProtoBuf.java index 2a2f8b77..ae7e4a6d 100644 --- a/src/com/google/common/io/protocol/ProtoBuf.java +++ b/src/com/google/common/io/protocol/ProtoBuf.java @@ -7,9 +7,7 @@ import java.io.*; import java.util.*; /** - * Protocol buffer message object. Currently, it is assumed that tags ids are - * not large. This could be improved by storing a start offset, reducing the - * assumption to a dense number space. + * Protocol buffer message object. * <p> * ProtoBuf instances may or may not reference a ProtoBufType instance, * representing information from a corresponding .proto file, which defines tag @@ -33,7 +31,6 @@ import java.util.*; * this behavior is that default values cannot be removed -- they would reappear * after a serialization cycle. If a tag has repeated values, setXXX(tag, value) * will overwrite all of them and getXXX(tag) will throw an exception. - * */ public class ProtoBuf { @@ -45,7 +42,9 @@ public class ProtoBuf { private static final String MSG_MISMATCH = "Type mismatch"; private static final String MSG_UNSUPPORTED = "Unsupp.Type"; - // names copied from //net/proto2/internal/wire_format.cc + // see + // http://code.google.com/apis/protocolbuffers/docs/overview.html + // for more details about wire format. static final int WIRETYPE_END_GROUP = 4; static final int WIRETYPE_FIXED32 = 5; static final int WIRETYPE_FIXED64 = 1; @@ -56,20 +55,19 @@ public class ProtoBuf { /** Maximum number of bytes for VARINT wire format (64 bit, 7 bit/byte) */ private static final int VARINT_MAX_BYTES = 10; - private static Long[] SMALL_NUMBERS = { - new Long(0), new Long(1), new Long(2), new Long(3), new Long(4), - new Long(5), new Long(6), new Long(7), new Long(8), new Long(9), - new Long(10), new Long(11), new Long(12), new Long(13), new Long(14), - new Long(15)}; - private ProtoBufType msgType; - private final Vector values = new Vector(); + private final IntMap values; /** * Wire types picked up on the wire or implied by setters (if no other * type information is available. */ - private final StringBuffer wireTypes = new StringBuffer(); + private final IntMap wireTypes; + + /** + * Saved by a call to #getCachedDataSize(false) and returned in #getCachedSize() + */ + private int cachedSize = Integer.MIN_VALUE; /** * Creates a protocol message according to the given description. The @@ -78,14 +76,22 @@ public class ProtoBuf { */ public ProtoBuf(ProtoBufType type) { this.msgType = type; + if (type != null) { + // if the type is known, use the type to create IntMaps. + values = type.newIntMapForProtoBuf(); + wireTypes = type.newIntMapForProtoBuf(); + } else { + values = new IntMap(); + wireTypes = new IntMap(); + } } - /** + /** * Clears all data stored in this ProtoBuf. */ public void clear() { - values.setSize(0); - wireTypes.setLength(0); + values.clear(); + wireTypes.clear(); } /** @@ -210,7 +216,7 @@ public class ProtoBuf { return (int) ((Long) getObject(tag, ProtoBufType.TYPE_INT32)).longValue(); } - /** + /** * Returns the integer value for the given repeated tag at the given index. */ public int getInt(int tag, int index) { @@ -305,7 +311,8 @@ public class ProtoBuf { * @param type the new type */ void setType(ProtoBufType type) { - if (values.size() != 0 || + // reject if the type is already set, or value is alreay set. + if (!values.isEmpty() || (msgType != null && type != null && type != msgType)) { throw new IllegalArgumentException(); } @@ -359,7 +366,8 @@ public class ProtoBuf { * @return this * @throws IOException raised if an IO exception occurs in the * underlying stream or the end of the stream is reached at - * an unexpected position + * an unexpected position, or if we encounter bad data + * while reading. */ public int parse(InputStream is, int available) throws IOException { @@ -376,11 +384,7 @@ public class ProtoBuf { break; } int tag = (int) (tagAndType >>> 3); - while (wireTypes.length() <= tag){ - wireTypes.append((char) ProtoBufType.TYPE_UNDEFINED); - } - wireTypes.setCharAt(tag, (char) wireType); - + wireTypes.put(tag, wireType); // first step: decode tag value Object value; switch (wireType) { @@ -390,8 +394,7 @@ public class ProtoBuf { if (isZigZagEncodedType(tag)) { v = zigZagDecode(v); } - value = (v >= 0 && v < SMALL_NUMBERS.length) ? - SMALL_NUMBERS[(int) v] : new Long(v); + value = v; break; // also used for fixed values @@ -408,9 +411,7 @@ public class ProtoBuf { shift += 8; } - value = (v >= 0 && v < SMALL_NUMBERS.length) - ? SMALL_NUMBERS[(int) v] - : new Long(v); + value = v; break; case WIRETYPE_LENGTH_DELIMITED: @@ -445,7 +446,8 @@ public class ProtoBuf { break; default: - throw new RuntimeException(MSG_UNSUPPORTED + wireType); + throw new IOException("Unknown wire type " + wireType + + ", reading garbage data?"); } insertObject(tag, getCount(tag), value); } @@ -466,9 +468,9 @@ public class ProtoBuf { throw new ArrayIndexOutOfBoundsException(); } if (count == 1){ - values.setElementAt(null, tag); + values.remove(tag); } else { - Vector v = (Vector) values.elementAt(tag); + Vector v = (Vector) values.get(tag); v.removeElementAt(index); } } @@ -477,12 +479,15 @@ public class ProtoBuf { * Returns the number of repeated and optional (0..1) values for a given tag. * Note: Default values are not counted (and in general not considered in * access methods for repeated tags), but considered for has(tag). + * + * @param tag the tag of the field + * @throws ArrayIndexOutOfBoundsException when tag is < 0 */ public int getCount(int tag) { - if (tag >= values.size()){ - return 0; + if (tag < 0) { + throw new ArrayIndexOutOfBoundsException(tag); } - Object o = values.elementAt(tag); + Object o = values.get(tag); if (o == null){ return 0; } @@ -502,38 +507,69 @@ public class ProtoBuf { tagType = msgType.getType(tag); } - if (tagType == ProtoBufType.TYPE_UNDEFINED && tag < wireTypes.length()) { - tagType = wireTypes.charAt(tag); + if (tagType == ProtoBufType.TYPE_UNDEFINED) { + Integer tagTypeObj = (Integer) wireTypes.get(tag); + if (tagTypeObj != null) { + tagType = tagTypeObj.intValue(); + } } - + if (tagType == ProtoBufType.TYPE_UNDEFINED && getCount(tag) > 0) { Object o = getObject(tag, 0, ProtoBufType.TYPE_UNDEFINED); - - tagType = (o instanceof Long) || (o instanceof Boolean) + + tagType = (o instanceof Long) || (o instanceof Boolean) ? WIRETYPE_VARINT : WIRETYPE_LENGTH_DELIMITED; } - + return tagType; } - + /** * Returns the number of bytes needed to store this protocol buffer */ public int getDataSize() { + return getCachedDataSize(false /* don't trust cache */); + } + + /** + * Each Protobuf keeps track of a <code> cachedSize </code> that is + * used to short circuit evaluation of its children's sizes. This value + * should only be trusted if you are reasonably certain it cannot be + * corrupt. (A corrupt cache can happen if any child ProtoBuf of this + * ProtoBuf has had a value set. In general, it is best to only trust + * the cache if you have just finished cleansing it.) + * + * <P/>The cache can be cleansed by calling this method with + * trustCache = false. + * + * @param trustCache if the cached size should be trusted. Set false to + * recompuate the size. + */ + private int getCachedDataSize(boolean trustCache) { + if (cachedSize != Integer.MIN_VALUE && trustCache) { + return cachedSize; + } int size = 0; - for (int tag = 0; tag <= maxTag(); tag++) { + IntMap.KeyIterator itr = values.keys(); + while(itr.hasNext()) { + int tag = itr.next(); for (int i = 0; i < getCount(tag); i++) { - size += getDataSize(tag, i); + size += getCachedDataSize(tag, i, trustCache); } - } - return size; + } + cachedSize = size; + + return cachedSize; } - - - /** - * Returns the size of the given value + + /** + * Returns the size of the child. + * + * @param tag tag used to determine the type of this child + * @param i used to determine which count this child is + * @param trustSizeCache passed down to #getCachedDataSize() */ - private int getDataSize(int tag, int i) { + private int getCachedDataSize(int tag, int i, boolean trustSizeCache) { int tagSize = getVarIntSize(tag << 3); switch(getWireType(tag)){ @@ -551,20 +587,20 @@ public class ProtoBuf { // take end group into account.... return tagSize + getProtoBuf(tag, i).getDataSize() + tagSize; } - + // take the object as stored Object o = getObject(tag, i, ProtoBufType.TYPE_UNDEFINED); - + int contentSize; - - if (o instanceof byte[]){ + + if (o instanceof byte[]) { contentSize = ((byte[]) o).length; } else if (o instanceof String) { contentSize = encodeUtf8((String) o, null, 0); } else { - contentSize = ((ProtoBuf) o).getDataSize(); + contentSize = ((ProtoBuf) o).getCachedDataSize(trustSizeCache); } - + return tagSize + getVarIntSize(contentSize) + contentSize; } @@ -584,67 +620,93 @@ public class ProtoBuf { return size; } - /** + /** * Writes this and nested protocol buffers to the given output stream. * * @param os target output stream - * @throws IOException thrown if there is an IOException + * @throws IOException thrown if there is an IOException */ public void outputTo(OutputStream os) throws IOException { - for (int tag = 0; tag <= maxTag(); tag++) { - int size = getCount(tag); - int wireType = getWireType(tag); - - // ignore default values - for (int i = 0; i < size; i++) { - writeVarInt(os, (tag << 3) | wireType); - - switch (wireType) { - case WIRETYPE_FIXED32: - case WIRETYPE_FIXED64: - long v = ((Long) getObject(tag, i, ProtoBufType.TYPE_INT64)) - .longValue(); - int cnt = (wireType == WIRETYPE_FIXED32) ? 4 : 8; - for (int b = 0; b < cnt; b++) { - os.write((int) (v & 0x0ff)); - v >>= 8; - } - break; + // We can't know what changed since we last output, so refresh the children. + getDataSize(); + outputToInternal(os); + } - case WIRETYPE_VARINT: - v = ((Long) getObject(tag, i, ProtoBufType.TYPE_INT64)).longValue(); - if (isZigZagEncodedType(tag)) { - v = zigZagEncode(v); - } - writeVarInt(os, v); - break; - - case WIRETYPE_LENGTH_DELIMITED: - Object o = getObject(tag, i, - getType(tag) == ProtoBufType.TYPE_MESSAGE - ? ProtoBufType.TYPE_UNDEFINED - : ProtoBufType.TYPE_DATA); - - if (o instanceof byte[]){ - byte[] data = (byte[]) o; - writeVarInt(os, data.length); - os.write(data); - } else { - ProtoBuf msg = (ProtoBuf) o; - writeVarInt(os, msg.getDataSize()); - msg.outputTo(os); - } - break; - - case WIRETYPE_START_GROUP: - ((ProtoBuf) getObject(tag, i, ProtoBufType.TYPE_GROUP)) - .outputTo(os); - writeVarInt(os, (tag << 3) | WIRETYPE_END_GROUP); - break; - - default: - throw new IllegalArgumentException(); - } + /** + * Recursive output method wrapped by #outputTo() + * + * @param os target output stream + * @throws IOException thrown if there is an IOException + */ + private void outputToInternal(OutputStream os) throws IOException { + IntMap.KeyIterator itr = values.keys(); + while (itr.hasNext()) { + int tag = itr.next(); + outputField(tag, os); + } + } + + /** + * Output a field indicated by the tag to given stream. + * + * @param tag the tag of the field to output. + * @param os target output stream + * @throws IOException thrown if there is an IOException + */ + private void outputField(int tag, OutputStream os) throws IOException { + int size = getCount(tag); + int wireType = getWireType(tag); + int wireTypeTag = (tag << 3) | wireType; + + // ignore default values + for (int i = 0; i < size; i++) { + writeVarInt(os, wireTypeTag); + switch (wireType) { + case WIRETYPE_FIXED32: + case WIRETYPE_FIXED64: + long v = ((Long) getObject(tag, i, ProtoBufType.TYPE_INT64)) + .longValue(); + int cnt = (wireType == WIRETYPE_FIXED32) ? 4 : 8; + for (int b = 0; b < cnt; b++) { + os.write((int) (v & 0x0ff)); + v >>= 8; + } + break; + + case WIRETYPE_VARINT: + v = ((Long) getObject(tag, i, ProtoBufType.TYPE_INT64)).longValue(); + if (isZigZagEncodedType(tag)) { + v = zigZagEncode(v); + } + writeVarInt(os, v); + break; + + case WIRETYPE_LENGTH_DELIMITED: + Object o = getObject(tag, i, + getType(tag) == ProtoBufType.TYPE_MESSAGE + ? ProtoBufType.TYPE_UNDEFINED + : ProtoBufType.TYPE_DATA); + + if (o instanceof byte[]) { + byte[] data = (byte[]) o; + writeVarInt(os, data.length); + os.write(data); + } else { + + ProtoBuf msg = (ProtoBuf) o; + writeVarInt(os, msg.getCachedDataSize(true)); + msg.outputToInternal(os); + } + break; + + case WIRETYPE_START_GROUP: + ((ProtoBuf) getObject(tag, i, ProtoBufType.TYPE_GROUP)) + .outputToInternal(os); + writeVarInt(os, (tag << 3) | WIRETYPE_END_GROUP); + break; + + default: + throw new IllegalArgumentException(); } } } @@ -693,12 +755,12 @@ public class ProtoBuf { outputTo(baos); return baos.toByteArray(); } - + /** * Returns the largest tag id used in this message (to simplify testing). */ public int maxTag() { - return values.size() - 1; + return values.maxKey(); } /** @@ -726,8 +788,7 @@ public class ProtoBuf { * Sets the given tag to the given long value. */ public void setLong(int tag, long value) { - setObject(tag, value >= 0 && value < SMALL_NUMBERS.length - ? SMALL_NUMBERS[(int) value] : new Long(value)); + setObject(tag, value); } /** @@ -795,8 +856,7 @@ public class ProtoBuf { * Inserts the given long value for the given tag at the given index. */ public void insertLong(int tag, int index, long value) { - insertObject(tag, index, value >= 0 && value < SMALL_NUMBERS.length - ? SMALL_NUMBERS[(int) value] : new Long(value)); + insertObject(tag, index, value); } /** @@ -879,8 +939,8 @@ public class ProtoBuf { case ProtoBufType.TYPE_GROUP: case ProtoBufType.TYPE_MESSAGE: if (msgType == null || msgType.getData(tag) == null || - ((ProtoBuf) object).msgType == null || - ((ProtoBuf) object).msgType == msgType.getData(tag)) { + ((ProtoBuf) object).msgType == null || + ((ProtoBuf) object).msgType.equals(msgType.getData(tag))) { return; } } @@ -944,7 +1004,7 @@ public class ProtoBuf { throw new ArrayIndexOutOfBoundsException(); } - Object o = values.elementAt(tag); + Object o = values.get(tag); Vector v = null; if (o instanceof Vector) { @@ -1025,14 +1085,14 @@ public class ProtoBuf { if (count == 0) { setObject(tag, o); } else { - Object curr = values.elementAt(tag); + Object curr = values.get(tag); Vector v; if (curr instanceof Vector) { v = (Vector) curr; } else { v = new Vector(); v.addElement(curr); - values.setElementAt(v, tag); + values.put(tag, v); } v.insertElementAt(o, index); } @@ -1042,7 +1102,7 @@ public class ProtoBuf { * Converts the object if a better suited class exists for the given .proto * type. If the formats are not compatible, an exception is thrown. */ - private Object convert(Object obj, int tagType) { + private static Object convert(Object obj, int tagType) { switch (tagType) { case ProtoBufType.TYPE_UNDEFINED: return obj; @@ -1068,7 +1128,7 @@ public class ProtoBuf { case ProtoBufType.TYPE_SINT32: case ProtoBufType.TYPE_SINT64: if (obj instanceof Boolean) { - return SMALL_NUMBERS[((Boolean) obj).booleanValue() ? 1 : 0]; + return ((Boolean) obj).booleanValue() ? 1 : 0; } return obj; case ProtoBufType.TYPE_DATA: @@ -1153,13 +1213,13 @@ public class ProtoBuf { * values. */ private void setObject(int tag, Object o) { - if (values.size() <= tag) { - values.setSize(tag + 1); + if (tag < 0) { + throw new ArrayIndexOutOfBoundsException(); } if (o != null) { assertTypeMatch(tag, o); } - values.setElementAt(o, tag); + values.put(tag, o); } /** |