From 839b180dbae98adf6caa54d0fb87b8d0a43081dc Mon Sep 17 00:00:00 2001 From: Feng Xiao Date: Mon, 24 Aug 2015 11:25:15 -0700 Subject: Cherry-pick Java utf8 change. --- .../main/java/com/google/protobuf/Descriptors.java | 13 +- .../java/com/google/protobuf/Proto3Utf8Test.java | 132 +++++++++++++++++++++ src/google/protobuf/compiler/java/java_helpers.h | 5 + .../protobuf/compiler/java/java_string_field.cc | 4 - .../compiler/java/java_string_field_lite.cc | 4 - 5 files changed, 149 insertions(+), 9 deletions(-) create mode 100644 java/src/main/java/com/google/protobuf/Proto3Utf8Test.java diff --git a/java/src/main/java/com/google/protobuf/Descriptors.java b/java/src/main/java/com/google/protobuf/Descriptors.java index fb9005bd..7cfc47f7 100644 --- a/java/src/main/java/com/google/protobuf/Descriptors.java +++ b/java/src/main/java/com/google/protobuf/Descriptors.java @@ -31,6 +31,7 @@ package com.google.protobuf; import com.google.protobuf.DescriptorProtos.*; +import com.google.protobuf.Descriptors.FileDescriptor.Syntax; import java.lang.ref.WeakReference; import java.util.ArrayList; @@ -912,7 +913,17 @@ public final class Descriptors { /** For internal use only. */ public boolean needsUtf8Check() { - return (type == Type.STRING) && (getFile().getOptions().getJavaStringCheckUtf8()); + if (type != Type.STRING) { + return false; + } + if (getContainingType().getOptions().getMapEntry()) { + // Always enforce strict UTF-8 checking for map fields. + return true; + } + if (getFile().getSyntax() == Syntax.PROTO3) { + return true; + } + return getFile().getOptions().getJavaStringCheckUtf8(); } public boolean isMapField() { diff --git a/java/src/main/java/com/google/protobuf/Proto3Utf8Test.java b/java/src/main/java/com/google/protobuf/Proto3Utf8Test.java new file mode 100644 index 00000000..4f9285be --- /dev/null +++ b/java/src/main/java/com/google/protobuf/Proto3Utf8Test.java @@ -0,0 +1,132 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import com.google.protobuf.testing.UnittestProto3Utf8.TestData; +import com.google.protobuf.testing.UnittestProto3Utf8.TestUtf8Checked; +import com.google.protobuf.testing.UnittestProto3Utf8.TestUtf8Unchecked; + +import junit.framework.TestCase; + +/** + * Test the UTF-8 checking semantics in proto3. + */ +public class Proto3Utf8Test extends TestCase { + + private static final ByteString INVALID_UTF8_BYTES = + ByteString.copyFrom(new byte[]{(byte) 0xff}); + private static final ByteString VALID_UTF8_BYTES = + ByteString.copyFrom(Internal.toByteArray("Hello world!")); + + private void assertAccepted(Message defaultInstance, ByteString data) + throws Exception { + defaultInstance.newBuilderForType().mergeFrom(data).build(); + } + + private void assertRejected(Message defaultInstance, ByteString data) + throws Exception { + try { + defaultInstance.newBuilderForType().mergeFrom(data).build(); + fail("Expects exceptions."); + } catch (Exception e) { + // Expected. + } + } + + private void assertAcceptedByUncheckedMessage(ByteString data) + throws Exception { + } + + private void assertRejectedByUncheckedMessage(ByteString data) + throws Exception { + } + + private void assertRejectedByCheckedMessage(ByteString data) + throws Exception { + assertRejected(TestUtf8Checked.getDefaultInstance(), data); + assertRejected(DynamicMessage.getDefaultInstance( + TestUtf8Checked.getDescriptor()), data); + } + + public void testOptionalFields() throws Exception { + TestData.Builder builder = TestData.newBuilder(); + builder.setOptionalValue(INVALID_UTF8_BYTES); + TestData data = builder.build(); + + assertAcceptedByUncheckedMessage(data.toByteString()); + assertRejectedByCheckedMessage(data.toByteString()); + } + + public void testRepeatedFields() throws Exception { + TestData.Builder builder = TestData.newBuilder(); + builder.addRepeatedValue(VALID_UTF8_BYTES); + builder.setOptionalValue(INVALID_UTF8_BYTES); + TestData data = builder.build(); + + assertAcceptedByUncheckedMessage(data.toByteString()); + assertRejectedByCheckedMessage(data.toByteString()); + } + + public void testOneofFields() throws Exception { + TestData.Builder builder = TestData.newBuilder(); + builder.setOneofValue(INVALID_UTF8_BYTES); + TestData data = builder.build(); + + assertAcceptedByUncheckedMessage(data.toByteString()); + assertRejectedByCheckedMessage(data.toByteString()); + } + + public void testMapKey() throws Exception { + TestData.Builder builder = TestData.newBuilder(); + builder.addMapValue( + TestData.MapValueEntry.newBuilder() + .setKey(INVALID_UTF8_BYTES) + .setValue(VALID_UTF8_BYTES).build()); + TestData data = builder.build(); + + // Map fields in Java are enforcing UTF-8 checking already. + assertRejectedByUncheckedMessage(data.toByteString()); + assertRejectedByCheckedMessage(data.toByteString()); + } + + public void testMapValue() throws Exception { + TestData.Builder builder = TestData.newBuilder(); + builder.addMapValue( + TestData.MapValueEntry.newBuilder() + .setKey(VALID_UTF8_BYTES) + .setValue(INVALID_UTF8_BYTES).build()); + TestData data = builder.build(); + + // Map fields in Java are enforcing UTF-8 checking already. + assertRejectedByUncheckedMessage(data.toByteString()); + assertRejectedByCheckedMessage(data.toByteString()); + } +} diff --git a/src/google/protobuf/compiler/java/java_helpers.h b/src/google/protobuf/compiler/java/java_helpers.h index 99ba6a18..7eef86a7 100644 --- a/src/google/protobuf/compiler/java/java_helpers.h +++ b/src/google/protobuf/compiler/java/java_helpers.h @@ -336,6 +336,11 @@ inline bool IsAnyMessage(const Descriptor* descriptor) { return descriptor->full_name() == "google.protobuf.Any"; } +inline bool CheckUtf8(const FieldDescriptor* descriptor) { + return descriptor->file()->syntax() == FileDescriptor::SYNTAX_PROTO3 || + descriptor->file()->options().java_string_check_utf8(); +} + } // namespace java } // namespace compiler } // namespace protobuf diff --git a/src/google/protobuf/compiler/java/java_string_field.cc b/src/google/protobuf/compiler/java/java_string_field.cc index 47e04659..72ebaeca 100644 --- a/src/google/protobuf/compiler/java/java_string_field.cc +++ b/src/google/protobuf/compiler/java/java_string_field.cc @@ -131,10 +131,6 @@ void SetPrimitiveVariables(const FieldDescriptor* descriptor, GenerateSetBitToLocal(messageBitIndex); } -bool CheckUtf8(const FieldDescriptor* descriptor) { - return descriptor->file()->options().java_string_check_utf8(); -} - } // namespace // =================================================================== diff --git a/src/google/protobuf/compiler/java/java_string_field_lite.cc b/src/google/protobuf/compiler/java/java_string_field_lite.cc index 032715b7..092e3c29 100644 --- a/src/google/protobuf/compiler/java/java_string_field_lite.cc +++ b/src/google/protobuf/compiler/java/java_string_field_lite.cc @@ -115,10 +115,6 @@ void SetPrimitiveVariables(const FieldDescriptor* descriptor, GenerateSetBitToLocal(messageBitIndex); } -bool CheckUtf8(const FieldDescriptor* descriptor) { - return descriptor->file()->options().java_string_check_utf8(); -} - } // namespace // =================================================================== -- cgit v1.2.3