Down integrate from Google internal branch for C++ and Java.

- Maps for C++ lite
- C++ Arena optimizations.
- Java Lite runtime code size optimization.

Change-Id: I7537a4357c1cb385d23f9e8aa7ffdfeefe079f13

12 files changed, 247 insertions, 5 deletions

diff --git a/java/src/test/java/com/google/protobuf/GeneratedMessageTest.java b/java/src/test/java/com/google/protobuf/GeneratedMessageTest.java
index 2d101ba7..2bd8d1a9 100644
--- a/java/src/test/java/com/google/protobuf/GeneratedMessageTest.java
+++ b/java/src/test/java/com/google/protobuf/GeneratedMessageTest.java
@@ -58,8 +58,8 @@ import protobuf_unittest.UnittestProto.ForeignMessage;
 import protobuf_unittest.UnittestProto.ForeignMessageOrBuilder;
 import protobuf_unittest.UnittestProto.NestedTestAllTypes;
 import protobuf_unittest.UnittestProto.TestAllExtensions;
-import protobuf_unittest.UnittestProto.TestAllTypes.NestedMessage;
 import protobuf_unittest.UnittestProto.TestAllTypes;
+import protobuf_unittest.UnittestProto.TestAllTypes.NestedMessage;
 import protobuf_unittest.UnittestProto.TestAllTypesOrBuilder;
 import protobuf_unittest.UnittestProto.TestExtremeDefaultValues;
 import protobuf_unittest.UnittestProto.TestOneof2;
diff --git a/java/src/test/java/com/google/protobuf/LazyFieldLiteTest.java b/java/src/test/java/com/google/protobuf/LazyFieldLiteTest.java
index e67c6d27..211b5697 100644
--- a/java/src/test/java/com/google/protobuf/LazyFieldLiteTest.java
+++ b/java/src/test/java/com/google/protobuf/LazyFieldLiteTest.java
@@ -30,6 +30,9 @@
 
 package com.google.protobuf;
 
+import static protobuf_unittest.UnittestProto.optionalInt32Extension;
+import static protobuf_unittest.UnittestProto.optionalInt64Extension;
+
 import protobuf_unittest.UnittestProto.TestAllExtensions;
 import protobuf_unittest.UnittestProto.TestAllTypes;
 
@@ -111,12 +114,146 @@ public class LazyFieldLiteTest extends TestCase {
     assertNotEqual(message.toByteString(), lazyField.toByteString());
   }
 
+  public void testMergeExtensions() throws Exception {
+    TestAllExtensions message = TestUtil.getAllExtensionsSet();
+    LazyFieldLite original = createLazyFieldLiteFromMessage(message);
+    LazyFieldLite merged = new LazyFieldLite();
+    merged.merge(original);
+    TestAllExtensions value = (TestAllExtensions) merged.getValue(
+        TestAllExtensions.getDefaultInstance());
+    assertEquals(message, value);
+  }
+
+  public void testEmptyLazyField() throws Exception {
+    LazyFieldLite field = new LazyFieldLite();
+    assertEquals(0, field.getSerializedSize());
+    assertEquals(ByteString.EMPTY, field.toByteString());
+  }
+
+  public void testInvalidProto() throws Exception {
+    // Silently fails and uses the default instance.
+    LazyFieldLite field = new LazyFieldLite(
+        TestUtil.getExtensionRegistry(), ByteString.copyFromUtf8("invalid"));
+    assertEquals(
+        TestAllTypes.getDefaultInstance(), field.getValue(TestAllTypes.getDefaultInstance()));
+    assertEquals(0, field.getSerializedSize());
+    assertEquals(ByteString.EMPTY, field.toByteString());
+  }
+
+  public void testMergeBeforeParsing() throws Exception {
+    TestAllTypes message1 = TestAllTypes.newBuilder().setOptionalInt32(1).build();
+    LazyFieldLite field1 = createLazyFieldLiteFromMessage(message1);
+    TestAllTypes message2 = TestAllTypes.newBuilder().setOptionalInt64(2).build();
+    LazyFieldLite field2 = createLazyFieldLiteFromMessage(message2);
+
+    field1.merge(field2);
+    TestAllTypes expected =
+        TestAllTypes.newBuilder().setOptionalInt32(1).setOptionalInt64(2).build();
+    assertEquals(expected, field1.getValue(TestAllTypes.getDefaultInstance()));
+  }
+
+  public void testMergeOneNotParsed() throws Exception {
+    // Test a few different paths that involve one message that was not parsed.
+    TestAllTypes message1 = TestAllTypes.newBuilder().setOptionalInt32(1).build();
+    TestAllTypes message2 = TestAllTypes.newBuilder().setOptionalInt64(2).build();
+    TestAllTypes expected =
+        TestAllTypes.newBuilder().setOptionalInt32(1).setOptionalInt64(2).build();
+
+    LazyFieldLite field1 = LazyFieldLite.fromValue(message1);
+    field1.getValue(TestAllTypes.getDefaultInstance());  // Force parsing.
+    LazyFieldLite field2 = createLazyFieldLiteFromMessage(message2);
+    field1.merge(field2);
+    assertEquals(expected, field1.getValue(TestAllTypes.getDefaultInstance()));
+
+    // Now reverse which one is parsed first.
+    field1 = LazyFieldLite.fromValue(message1);
+    field2 = createLazyFieldLiteFromMessage(message2);
+    field2.getValue(TestAllTypes.getDefaultInstance());  // Force parsing.
+    field1.merge(field2);
+    assertEquals(expected, field1.getValue(TestAllTypes.getDefaultInstance()));
+  }
+
+  public void testMergeInvalid() throws Exception {
+    // Test a few different paths that involve one message that was not parsed.
+    TestAllTypes message = TestAllTypes.newBuilder().setOptionalInt32(1).build();
+    LazyFieldLite valid = LazyFieldLite.fromValue(message);
+    LazyFieldLite invalid = new LazyFieldLite(
+        TestUtil.getExtensionRegistry(), ByteString.copyFromUtf8("invalid"));
+    invalid.merge(valid);
+
+    // We swallow the exception and just use the set field.
+    assertEquals(message, invalid.getValue(TestAllTypes.getDefaultInstance()));
+  }
+
+  public void testMergeKeepsExtensionsWhenPossible() throws Exception {
+    // In this test we attempt to only use the empty registry, which will strip out all extensions
+    // when serializing and then parsing. We verify that each code path will attempt to not
+    // serialize and parse a message that was set directly without going through the
+    // extensionRegistry.
+    TestAllExtensions messageWithExtensions =
+        TestAllExtensions.newBuilder().setExtension(optionalInt32Extension, 42).build();
+    TestAllExtensions emptyMessage = TestAllExtensions.newBuilder().build();
+
+    ExtensionRegistryLite emptyRegistry = ExtensionRegistryLite.getEmptyRegistry();
+
+    LazyFieldLite field = LazyFieldLite.fromValue(messageWithExtensions);
+    field.merge(createLazyFieldLiteFromMessage(emptyRegistry, emptyMessage));
+    assertEquals(messageWithExtensions, field.getValue(TestAllExtensions.getDefaultInstance()));
+
+    // Now reverse the order of the merging.
+    field = createLazyFieldLiteFromMessage(emptyRegistry, emptyMessage);
+    field.merge(LazyFieldLite.fromValue(messageWithExtensions));
+    assertEquals(messageWithExtensions, field.getValue(TestAllExtensions.getDefaultInstance()));
+
+    // Now try parsing the empty field first.
+    field = LazyFieldLite.fromValue(messageWithExtensions);
+    LazyFieldLite other = createLazyFieldLiteFromMessage(emptyRegistry, emptyMessage);
+    other.getValue(TestAllExtensions.getDefaultInstance());  // Force parsing.
+    field.merge(other);
+    assertEquals(messageWithExtensions, field.getValue(TestAllExtensions.getDefaultInstance()));
+
+    // And again reverse.
+    field = createLazyFieldLiteFromMessage(emptyRegistry, emptyMessage);
+    field.getValue(TestAllExtensions.getDefaultInstance());  // Force parsing.
+    other = LazyFieldLite.fromValue(messageWithExtensions);
+    field.merge(other);
+    assertEquals(messageWithExtensions, field.getValue(TestAllExtensions.getDefaultInstance()));
+  }
+
+  public void testMergeMightLoseExtensions() throws Exception {
+    // Test that we don't know about the extensions when parsing.
+    TestAllExtensions message1 =
+        TestAllExtensions.newBuilder().setExtension(optionalInt32Extension, 1).build();
+    TestAllExtensions message2 =
+        TestAllExtensions.newBuilder().setExtension(optionalInt64Extension, 2L).build();
+
+    LazyFieldLite field = LazyFieldLite.fromValue(message1);
+    field.merge(LazyFieldLite.fromValue(message2));
+
+    // We lose the extensions from message 2 because we have to serialize it and then parse it
+    // again, using the empty registry this time.
+    TestAllExtensions value =
+        (TestAllExtensions) field.getValue(TestAllExtensions.getDefaultInstance());
+    assertTrue(value.hasExtension(optionalInt32Extension));
+    assertEquals(Integer.valueOf(1), value.getExtension(optionalInt32Extension));
+    assertFalse(value.hasExtension(optionalInt64Extension));
+
+    // The field is still there, it is just unknown.
+    assertTrue(value.getUnknownFields()
+        .hasField(optionalInt64Extension.getDescriptor().getNumber()));
+  }
+
 
   // Help methods.
 
   private LazyFieldLite createLazyFieldLiteFromMessage(MessageLite message) {
+    return createLazyFieldLiteFromMessage(TestUtil.getExtensionRegistry(), message);
+  }
+
+  private LazyFieldLite createLazyFieldLiteFromMessage(
+      ExtensionRegistryLite extensionRegistry, MessageLite message) {
     ByteString bytes = message.toByteString();
-    return new LazyFieldLite(TestUtil.getExtensionRegistry(), bytes);
+    return new LazyFieldLite(extensionRegistry, bytes);
   }
 
   private void changeValue(LazyFieldLite lazyField) {
diff --git a/java/src/test/java/com/google/protobuf/LazyMessageLiteTest.java b/java/src/test/java/com/google/protobuf/LazyMessageLiteTest.java
index 9de794fe..afe0fffd 100644
--- a/java/src/test/java/com/google/protobuf/LazyMessageLiteTest.java
+++ b/java/src/test/java/com/google/protobuf/LazyMessageLiteTest.java
@@ -30,6 +30,7 @@
 
 package com.google.protobuf;
 
+import protobuf_unittest.LazyFieldsLite.LazyExtension;
 import protobuf_unittest.LazyFieldsLite.LazyInnerMessageLite;
 import protobuf_unittest.LazyFieldsLite.LazyMessageLite;
 import protobuf_unittest.LazyFieldsLite.LazyNestedInnerMessageLite;
@@ -285,4 +286,22 @@ public class LazyMessageLiteTest extends TestCase {
 
     assertEquals(bytes, deserialized.toByteString());
   }
+
+  public void testExtensions() throws Exception {
+    LazyInnerMessageLite.Builder innerBuilder = LazyInnerMessageLite.newBuilder();
+    innerBuilder.setExtension(
+        LazyExtension.extension, LazyExtension.newBuilder()
+        .setName("name").build());
+    assertTrue(innerBuilder.hasExtension(LazyExtension.extension));
+    assertEquals("name", innerBuilder.getExtension(LazyExtension.extension).getName());
+
+    LazyInnerMessageLite innerMessage = innerBuilder.build();
+    assertTrue(innerMessage.hasExtension(LazyExtension.extension));
+    assertEquals("name", innerMessage.getExtension(LazyExtension.extension).getName());
+
+    LazyMessageLite lite = LazyMessageLite.newBuilder()
+        .setInner(innerMessage).build();
+    assertTrue(lite.getInner().hasExtension(LazyExtension.extension));
+    assertEquals("name", lite.getInner().getExtension(LazyExtension.extension).getName());
+  }
 }
diff --git a/java/src/test/java/com/google/protobuf/LiteralByteStringTest.java b/java/src/test/java/com/google/protobuf/LiteralByteStringTest.java
index ff39ca3f..046832de 100644
--- a/java/src/test/java/com/google/protobuf/LiteralByteStringTest.java
+++ b/java/src/test/java/com/google/protobuf/LiteralByteStringTest.java
@@ -289,7 +289,6 @@ public class LiteralByteStringTest extends TestCase {
     assertEquals("Output.reset() resets the output", 0, output.size());
     assertEquals("Output.reset() resets the output",
         ByteString.EMPTY, output.toByteString());
-    
   }
 
   public void testToString() throws UnsupportedEncodingException {
@@ -299,6 +298,27 @@ public class LiteralByteStringTest extends TestCase {
     assertEquals(classUnderTest + " unicode must match", testString, roundTripString);
   }
 
+  public void testToString_returnsCanonicalEmptyString() throws UnsupportedEncodingException{
+    assertSame(classUnderTest + " must be the same string references",
+        ByteString.EMPTY.toString(UTF_8), new LiteralByteString(new byte[]{}).toString(UTF_8));
+  }
+
+  public void testToString_raisesException() throws UnsupportedEncodingException{
+    try {
+      ByteString.EMPTY.toString("invalid");
+      fail("Should have thrown an exception.");
+    } catch (UnsupportedEncodingException expected) {
+      // This is success
+    }
+
+    try {
+      new LiteralByteString(referenceBytes).toString("invalid");
+      fail("Should have thrown an exception.");
+    } catch (UnsupportedEncodingException expected) {
+      // This is success
+    }
+  }
+
   public void testEquals() {
     assertEquals(classUnderTest + " must not equal null", false, stringUnderTest.equals(null));
     assertEquals(classUnderTest + " must equal self", stringUnderTest, stringUnderTest);
@@ -311,7 +331,7 @@ public class LiteralByteStringTest extends TestCase {
 
     byte[] mungedBytes = new byte[referenceBytes.length];
     System.arraycopy(referenceBytes, 0, mungedBytes, 0, referenceBytes.length);
-    mungedBytes[mungedBytes.length - 5] ^= 0xFF;
+    mungedBytes[mungedBytes.length - 5] = (byte) (mungedBytes[mungedBytes.length - 5] ^ 0xFF);
     assertFalse(classUnderTest + " must not equal every string with the same length",
         stringUnderTest.equals(new LiteralByteString(mungedBytes)));
   }
diff --git a/java/src/test/java/com/google/protobuf/MapForProto2Test.java b/java/src/test/java/com/google/protobuf/MapForProto2Test.java
index 33ba7150..78cba1b4 100644
--- a/java/src/test/java/com/google/protobuf/MapForProto2Test.java
+++ b/java/src/test/java/com/google/protobuf/MapForProto2Test.java
@@ -34,6 +34,7 @@ import com.google.protobuf.Descriptors.FieldDescriptor;
 import map_test.MapForProto2TestProto.TestMap;
 import map_test.MapForProto2TestProto.TestMap.MessageValue;
 import map_test.MapForProto2TestProto.TestMap.MessageWithRequiredFields;
+import map_test.MapForProto2TestProto.TestRecursiveMap;
 import map_test.MapForProto2TestProto.TestUnknownEnumValue;
 
 import junit.framework.TestCase;
@@ -499,4 +500,17 @@ public class MapForProto2Test extends TestCase {
     message = builder.build();
     assertTrue(message.isInitialized());
   }
+
+  public void testRecursiveMap() throws Exception {
+    TestRecursiveMap.Builder builder = TestRecursiveMap.newBuilder();
+    builder.getMutableRecursiveMapField().put(
+        1, TestRecursiveMap.newBuilder().setValue(2).build());
+    builder.getMutableRecursiveMapField().put(
+        3, TestRecursiveMap.newBuilder().setValue(4).build());
+    ByteString data = builder.build().toByteString();
+
+    TestRecursiveMap message = TestRecursiveMap.parseFrom(data);
+    assertEquals(2, message.getRecursiveMapField().get(1).getValue());
+    assertEquals(4, message.getRecursiveMapField().get(3).getValue());
+  }
 }
diff --git a/java/src/test/java/com/google/protobuf/MapTest.java b/java/src/test/java/com/google/protobuf/MapTest.java
index 6a1e9078..b8e67b7c 100644
--- a/java/src/test/java/com/google/protobuf/MapTest.java
+++ b/java/src/test/java/com/google/protobuf/MapTest.java
@@ -269,7 +269,6 @@ public class MapTest extends TestCase {
     assertFalse(m2.equals(m1));
   }
 
-
   public void testNestedBuilderOnChangeEventPropagation() {
     TestOnChangeEventPropagation.Builder parent =
         TestOnChangeEventPropagation.newBuilder();
diff --git a/java/src/test/java/com/google/protobuf/RopeByteStringTest.java b/java/src/test/java/com/google/protobuf/RopeByteStringTest.java
index b3970196..0f2344d6 100644
--- a/java/src/test/java/com/google/protobuf/RopeByteStringTest.java
+++ b/java/src/test/java/com/google/protobuf/RopeByteStringTest.java
@@ -118,6 +118,34 @@ public class RopeByteStringTest extends LiteralByteStringTest {
         flatString.hashCode(), unicode.hashCode());
   }
 
+  @Override
+  public void testToString_returnsCanonicalEmptyString() throws UnsupportedEncodingException {
+    RopeByteString ropeByteString =
+        RopeByteString.newInstanceForTest(ByteString.EMPTY, ByteString.EMPTY);
+    assertSame(classUnderTest + " must be the same string references",
+        ByteString.EMPTY.toString(UTF_8), ropeByteString.toString(UTF_8));
+  }
+
+  public void testToString_raisesException() throws UnsupportedEncodingException{
+    try {
+      ByteString byteString =
+          RopeByteString.newInstanceForTest(ByteString.EMPTY, ByteString.EMPTY);
+      byteString.toString("invalid");
+      fail("Should have thrown an exception.");
+    } catch (UnsupportedEncodingException expected) {
+      // This is success
+    }
+
+    try {
+      ByteString byteString = RopeByteString.concatenate(ByteString.copyFromUtf8("foo"),
+          ByteString.copyFromUtf8("bar"));
+      byteString.toString("invalid");
+      fail("Should have thrown an exception.");
+    } catch (UnsupportedEncodingException expected) {
+      // This is success
+    }
+  }
+
   public void testJavaSerialization() throws Exception {
     ByteArrayOutputStream out = new ByteArrayOutputStream();
     ObjectOutputStream oos = new ObjectOutputStream(out);
diff --git a/java/src/test/java/com/google/protobuf/lazy_fields_lite.proto b/java/src/test/java/com/google/protobuf/lazy_fields_lite.proto
index 015dc267..5580f72d 100644
--- a/java/src/test/java/com/google/protobuf/lazy_fields_lite.proto
+++ b/java/src/test/java/com/google/protobuf/lazy_fields_lite.proto
@@ -54,6 +54,15 @@ message LazyInnerMessageLite {
   optional int32 num = 1;
   optional int32 num_with_default = 2 [default = 42];
   optional LazyNestedInnerMessageLite nested = 3 [lazy = true];
+
+  extensions 1000 to max;
+}
+
+message LazyExtension {
+  extend LazyInnerMessageLite {
+    optional LazyExtension extension = 1000;
+  }
+  optional string name = 1;
 }
 
 message LazyNestedInnerMessageLite {
diff --git a/java/src/test/java/com/google/protobuf/map_for_proto2_lite_test.proto b/java/src/test/java/com/google/protobuf/map_for_proto2_lite_test.proto
index a1fe856c..d5418f28 100644
--- a/java/src/test/java/com/google/protobuf/map_for_proto2_lite_test.proto
+++ b/java/src/test/java/com/google/protobuf/map_for_proto2_lite_test.proto
@@ -63,6 +63,13 @@ message TestUnknownEnumValue {
   // parsing behavior of TestMap regarding unknown enum values.
   map<int32, int32> int32_to_int32_field = 4;
 }
+
+// Test that the maps initialization code works correctly when the map field
+// references the containing message.
+message TestRecursiveMap {
+  optional int32 value = 1;
+  map<int32, TestRecursiveMap> recursive_map_field = 2;
+}
 package map_for_proto2_lite_test;
 option java_package = "map_lite_test";
 option optimize_for = LITE_RUNTIME;
diff --git a/java/src/test/java/com/google/protobuf/map_for_proto2_test.proto b/java/src/test/java/com/google/protobuf/map_for_proto2_test.proto
index a0ec7ac5..a9be5166 100644
--- a/java/src/test/java/com/google/protobuf/map_for_proto2_test.proto
+++ b/java/src/test/java/com/google/protobuf/map_for_proto2_test.proto
@@ -65,3 +65,10 @@ message TestUnknownEnumValue {
   // parsing behavior of TestMap regarding unknown enum values.
   map<int32, int32> int32_to_int32_field = 4;
 }
+
+// Test that the maps initialization code works correctly when the map field
+// references the containing message.
+message TestRecursiveMap {
+  optional int32 value = 1;
+  map<int32, TestRecursiveMap> recursive_map_field = 2;
+}
diff --git a/java/src/test/java/com/google/protobuf/map_test.proto b/java/src/test/java/com/google/protobuf/map_test.proto
index 105ee3f8..bf692c22 100644
--- a/java/src/test/java/com/google/protobuf/map_test.proto
+++ b/java/src/test/java/com/google/protobuf/map_test.proto
@@ -36,6 +36,7 @@ option java_package = "map_test";
 option java_outer_classname = "MapTestProto";
 option java_generate_equals_and_hash = true;
 
+
 message TestMap {
   message MessageValue {
     optional int32 value = 1;
diff --git a/java/src/test/java/com/google/protobuf/test_bad_identifiers.proto b/java/src/test/java/com/google/protobuf/test_bad_identifiers.proto
index 67035fd5..dc082615 100644
--- a/java/src/test/java/com/google/protobuf/test_bad_identifiers.proto
+++ b/java/src/test/java/com/google/protobuf/test_bad_identifiers.proto
@@ -45,6 +45,7 @@ option java_package = "com.google.protobuf";
 option java_outer_classname = "TestBadIdentifiersProto";
 option java_generate_equals_and_hash = true;
 
+
 message TestMessage {
   optional string cached_size = 1;
   optional string serialized_size = 2;