diff options
48 files changed, 3406 insertions, 53 deletions
@@ -101,7 +101,11 @@ objectivec/ProtocolBuffers_iOS.xcodeproj/xcuserdata/ conformance/.libs/ conformance/com/ conformance/conformance-cpp +conformance/conformance-csharp +conformance/conformance-java conformance/conformance-test-runner conformance/conformance.pb.cc conformance/conformance.pb.h +conformance/conformance.rb +conformance/javac_middleman conformance/protoc_middleman diff --git a/.travis.yml b/.travis.yml index 4edf3b30..7be0d18e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,8 @@ language: cpp os: - linux - osx +# The Objective C build needs Xcode 6.4 or later. +osx_image: xcode7.1 script: - ./travis.sh $CONFIG env: @@ -46,11 +48,22 @@ matrix: # which doesn't work on OS X. - os: osx env: CONFIG=csharp + # Add into the matrix OS X tests of Objective C (needs Xcode, so it won't + # work on other platforms). These are split so it doesn't take as long to run. + include: + - os: osx + env: CONFIG=objectivec_ios + - os: osx + env: CONFIG=objectivec_osx allow_failures: # These currently do not work on OS X but are being worked on by @haberman. - os: osx env: CONFIG=ruby22 - os: osx env: CONFIG=jruby + # Travis seems be flaky in letting the iOS simulator launch, so keep that + # flake from failing builds by marking that build as an allowed failure. + - os: osx + env: CONFIG=objectivec_ios notifications: email: false @@ -5,7 +5,7 @@ This license applies to all parts of Protocol Buffers except the following: This file is copyrighted by Red Hat Inc. - Atomicops support for AIX/POWER, located in - src/google/protobuf/stubs/atomicops_internals_aix.h. + src/google/protobuf/stubs/atomicops_internals_power.h. This file is copyrighted by Bloomberg Finance LP. Copyright 2014, Google Inc. All rights reserved. diff --git a/Makefile.am b/Makefile.am index 62233012..694d73a1 100644 --- a/Makefile.am +++ b/Makefile.am @@ -35,6 +35,10 @@ clean-local: if test -e conformance/Makefile; then \ echo "Making clean in conformance"; \ cd conformance && $(MAKE) $(AM_MAKEFLAGS) clean; \ + fi; \ + if test -e objectivec/DevTools; then \ + echo "Cleaning any ObjC pyc files"; \ + rm -f objectivec/DevTools/*.pyc; \ fi pkgconfigdir = $(libdir)/pkgconfig @@ -82,6 +86,8 @@ csharp_EXTRA_DIST= \ csharp/src/Google.Protobuf.Test/Google.Protobuf.Test.csproj \ csharp/src/Google.Protobuf.Test/IssuesTest.cs \ csharp/src/Google.Protobuf.Test/JsonFormatterTest.cs \ + csharp/src/Google.Protobuf.Test/JsonParserTest.cs \ + csharp/src/Google.Protobuf.Test/JsonTokenizerTest.cs \ csharp/src/Google.Protobuf.Test/Properties/AppManifest.xml \ csharp/src/Google.Protobuf.Test/Properties/AssemblyInfo.cs \ csharp/src/Google.Protobuf.Test/Reflection/DescriptorsTest.cs \ @@ -117,8 +123,12 @@ csharp_EXTRA_DIST= \ csharp/src/Google.Protobuf/Google.Protobuf.nuspec \ csharp/src/Google.Protobuf/IDeepCloneable.cs \ csharp/src/Google.Protobuf/IMessage.cs \ + csharp/src/Google.Protobuf/InvalidJsonException.cs \ csharp/src/Google.Protobuf/InvalidProtocolBufferException.cs \ csharp/src/Google.Protobuf/JsonFormatter.cs \ + csharp/src/Google.Protobuf/JsonParser.cs \ + csharp/src/Google.Protobuf/JsonToken.cs \ + csharp/src/Google.Protobuf/JsonTokenizer.cs \ csharp/src/Google.Protobuf/LimitedInputStream.cs \ csharp/src/Google.Protobuf/MessageExtensions.cs \ csharp/src/Google.Protobuf/MessageParser.cs \ diff --git a/cmake/extract_includes.bat.in b/cmake/extract_includes.bat.in index eaf97583..f6d8b893 100644 --- a/cmake/extract_includes.bat.in +++ b/cmake/extract_includes.bat.in @@ -72,7 +72,7 @@ copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\source_context.pb.h in copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\struct.pb.h include\google\protobuf\struct.pb.h copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomic_sequence_num.h include\google\protobuf\stubs\atomic_sequence_num.h copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops.h include\google\protobuf\stubs\atomicops.h -copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_aix.h include\google\protobuf\stubs\atomicops_internals_aix.h +copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_power.h include\google\protobuf\stubs\atomicops_internals_power.h copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_arm64_gcc.h include\google\protobuf\stubs\atomicops_internals_arm64_gcc.h copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_arm_gcc.h include\google\protobuf\stubs\atomicops_internals_arm_gcc.h copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\atomicops_internals_arm_qnx.h include\google\protobuf\stubs\atomicops_internals_arm_qnx.h diff --git a/conformance/Makefile.am b/conformance/Makefile.am index c74eb1c8..b6fda2a8 100644 --- a/conformance/Makefile.am +++ b/conformance/Makefile.am @@ -7,6 +7,10 @@ protoc_outputs = \ conformance.pb.cc \ conformance.pb.h +other_language_protoc_outputs = \ + conformance.rb \ + com/google/protobuf/conformance/Conformance.java + bin_PROGRAMS = conformance-test-runner conformance-cpp conformance_test_runner_LDADD = $(top_srcdir)/src/libprotobuf.la @@ -38,14 +42,16 @@ endif $(protoc_outputs): protoc_middleman +$(other_language_protoc_outputs): protoc_middleman + BUILT_SOURCES = $(protoc_outputs) -CLEANFILES = $(protoc_outputs) protoc_middleman javac_middleman conformance-java conformance-csharp +CLEANFILES = $(protoc_outputs) protoc_middleman javac_middleman conformance-java conformance-csharp $(other_language_protoc_outputs) MAINTAINERCLEANFILES = \ Makefile.in -javac_middleman: ConformanceJava.java protoc_middleman +javac_middleman: ConformanceJava.java protoc_middleman $(other_language_protoc_outputs) javac -classpath ../java/target/classes ConformanceJava.java com/google/protobuf/conformance/Conformance.java @touch javac_middleman @@ -58,7 +64,7 @@ conformance-java: javac_middleman # Currently the conformance code is alongside the rest of the C# # source, as it's easier to maintain there. We assume we've already # built that, so we just need a script to run it. -conformance-csharp: +conformance-csharp: $(other_language_protoc_outputs) @echo "Writing shortcut script conformance-csharp..." @echo '#! /bin/sh' > conformance-csharp @echo 'mono ../csharp/src/Google.Protobuf.Conformance/bin/Release/Google.Protobuf.Conformance.exe "$$@"' >> conformance-csharp @@ -74,5 +80,5 @@ test_java: protoc_middleman conformance-test-runner conformance-java test_csharp: protoc_middleman conformance-test-runner conformance-csharp ./conformance-test-runner --failure_list failure_list_csharp.txt ./conformance-csharp -test_ruby: protoc_middleman conformance-test-runner +test_ruby: protoc_middleman conformance-test-runner $(other_language_protoc_outputs) RUBYLIB=../ruby/lib:. ./conformance-test-runner --failure_list failure_list_ruby.txt ./conformance_ruby.rb diff --git a/csharp/src/Google.Protobuf.Conformance/Conformance.cs b/csharp/src/Google.Protobuf.Conformance/Conformance.cs index 9626bea3..3ebaa4ef 100644 --- a/csharp/src/Google.Protobuf.Conformance/Conformance.cs +++ b/csharp/src/Google.Protobuf.Conformance/Conformance.cs @@ -165,6 +165,7 @@ namespace Conformance { #region Messages /// <summary> /// Represents a single test case's input. The testee should: + /// /// 1. parse this proto (which should always succeed) /// 2. parse the protobuf or JSON payload in "payload" (which may fail) /// 3. if the parse succeeded, serialize the message in the requested format. @@ -410,6 +411,7 @@ namespace Conformance { /// <summary> /// This string should be set to indicate parsing failed. The string can /// provide more information about the parse error if it is available. + /// /// Setting this string does not necessarily mean the testee failed the /// test. Some of the test cases are intentionally invalid input. /// </summary> diff --git a/csharp/src/Google.Protobuf.Test/CodedInputStreamTest.cs b/csharp/src/Google.Protobuf.Test/CodedInputStreamTest.cs index 54c44e47..6ae02112 100644 --- a/csharp/src/Google.Protobuf.Test/CodedInputStreamTest.cs +++ b/csharp/src/Google.Protobuf.Test/CodedInputStreamTest.cs @@ -284,7 +284,7 @@ namespace Google.Protobuf Assert.Throws<InvalidProtocolBufferException>(() => input.ReadBytes());
}
- private static TestRecursiveMessage MakeRecursiveMessage(int depth)
+ internal static TestRecursiveMessage MakeRecursiveMessage(int depth)
{
if (depth == 0)
{
@@ -296,7 +296,7 @@ namespace Google.Protobuf }
}
- private static void AssertMessageDepth(TestRecursiveMessage message, int depth)
+ internal static void AssertMessageDepth(TestRecursiveMessage message, int depth)
{
if (depth == 0)
{
diff --git a/csharp/src/Google.Protobuf.Test/Google.Protobuf.Test.csproj b/csharp/src/Google.Protobuf.Test/Google.Protobuf.Test.csproj index 33be5dae..cd15b968 100644 --- a/csharp/src/Google.Protobuf.Test/Google.Protobuf.Test.csproj +++ b/csharp/src/Google.Protobuf.Test/Google.Protobuf.Test.csproj @@ -95,6 +95,8 @@ <Compile Include="Collections\MapFieldTest.cs" />
<Compile Include="Collections\RepeatedFieldTest.cs" />
<Compile Include="JsonFormatterTest.cs" />
+ <Compile Include="JsonParserTest.cs" />
+ <Compile Include="JsonTokenizerTest.cs" />
<Compile Include="Reflection\DescriptorsTest.cs" />
<Compile Include="Reflection\FieldAccessTest.cs" />
<Compile Include="SampleEnum.cs" />
diff --git a/csharp/src/Google.Protobuf.Test/JsonFormatterTest.cs b/csharp/src/Google.Protobuf.Test/JsonFormatterTest.cs index 3e7a3d2e..08cedad8 100644 --- a/csharp/src/Google.Protobuf.Test/JsonFormatterTest.cs +++ b/csharp/src/Google.Protobuf.Test/JsonFormatterTest.cs @@ -276,6 +276,13 @@ namespace Google.Protobuf } [Test] + public void WrapperFormatting_Message() + { + Assert.AreEqual("\"\"", JsonFormatter.Default.Format(new StringValue())); + Assert.AreEqual("0", JsonFormatter.Default.Format(new Int32Value())); + } + + [Test] public void WrapperFormatting_IncludeNull() { // The actual JSON here is very large because there are lots of fields. Just test a couple of them. @@ -376,12 +383,12 @@ namespace Google.Protobuf { Fields = { - { "a", new Value { NullValue = new NullValue() } }, - { "b", new Value { BoolValue = false } }, - { "c", new Value { NumberValue = 10.5 } }, - { "d", new Value { StringValue = "text" } }, - { "e", new Value { ListValue = new ListValue { Values = { new Value { StringValue = "t1" }, new Value { NumberValue = 5 } } } } }, - { "f", new Value { StructValue = new Struct { Fields = { { "nested", new Value { StringValue = "value" } } } } } } + { "a", Value.ForNull() }, + { "b", Value.ForBool(false) }, + { "c", Value.ForNumber(10.5) }, + { "d", Value.ForString("text") }, + { "e", Value.ForList(Value.ForString("t1"), Value.ForNumber(5)) }, + { "f", Value.ForStruct(new Struct { Fields = { { "nested", Value.ForString("value") } } }) } } }; AssertJson("{ 'a': null, 'b': false, 'c': 10.5, 'd': 'text', 'e': [ 't1', 5 ], 'f': { 'nested': 'value' } }", message.ToString()); @@ -405,6 +412,14 @@ namespace Google.Protobuf AssertJson("{ 'fieldMaskField': 'user.displayName,photo' }", JsonFormatter.Default.Format(message)); } + // SourceContext is an example of a well-known type with no special JSON handling + [Test] + public void SourceContextStandalone() + { + var message = new SourceContext { FileName = "foo.proto" }; + AssertJson("{ 'fileName': 'foo.proto' }", JsonFormatter.Default.Format(message)); + } + /// <summary> /// Checks that the actual JSON is the same as the expected JSON - but after replacing /// all apostrophes in the expected JSON with double quotes. This basically makes the tests easier diff --git a/csharp/src/Google.Protobuf.Test/JsonParserTest.cs b/csharp/src/Google.Protobuf.Test/JsonParserTest.cs new file mode 100644 index 00000000..c48b151d --- /dev/null +++ b/csharp/src/Google.Protobuf.Test/JsonParserTest.cs @@ -0,0 +1,745 @@ +#region Copyright notice and license +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#endregion + +using Google.Protobuf.TestProtos; +using Google.Protobuf.WellKnownTypes; +using NUnit.Framework; +using System; + +namespace Google.Protobuf +{ + /// <summary> + /// Unit tests for JSON parsing. Some tests are ignored at the moment as the desired behaviour + /// isn't fully known, either in terms of which exceptions should be thrown or whether they should + /// count as valid values. + /// </summary> + public class JsonParserTest + { + // Sanity smoke test + [Test] + public void AllTypesRoundtrip() + { + AssertRoundtrip(SampleMessages.CreateFullTestAllTypes()); + } + + [Test] + public void Maps() + { + AssertRoundtrip(new TestMap { MapStringString = { { "with spaces", "bar" }, { "a", "b" } } }); + AssertRoundtrip(new TestMap { MapInt32Int32 = { { 0, 1 }, { 2, 3 } } }); + AssertRoundtrip(new TestMap { MapBoolBool = { { false, true }, { true, false } } }); + } + + [Test] + [TestCase(" 1 ")] + [TestCase("+1")] + [TestCase("1,000")] + [TestCase("1.5")] + public void IntegerMapKeysAreStrict(string keyText) + { + // Test that integer parsing is strict. We assume that if this is correct for int32, + // it's correct for other numeric key types. + var json = "{ \"mapInt32Int32\": { \"" + keyText + "\" : \"1\" } }"; + Assert.Throws<InvalidProtocolBufferException>(() => JsonParser.Default.Parse<TestMap>(json)); + } + + [Test] + public void SourceContextRoundtrip() + { + AssertRoundtrip(new SourceContext { FileName = "foo.proto" }); + } + + [Test] + public void SingularWrappers_DefaultNonNullValues() + { + var message = new TestWellKnownTypes + { + StringField = "", + BytesField = ByteString.Empty, + BoolField = false, + FloatField = 0f, + DoubleField = 0d, + Int32Field = 0, + Int64Field = 0, + Uint32Field = 0, + Uint64Field = 0 + }; + AssertRoundtrip(message); + } + + [Test] + public void SingularWrappers_NonDefaultValues() + { + var message = new TestWellKnownTypes + { + StringField = "x", + BytesField = ByteString.CopyFrom(1, 2, 3), + BoolField = true, + FloatField = 12.5f, + DoubleField = 12.25d, + Int32Field = 1, + Int64Field = 2, + Uint32Field = 3, + Uint64Field = 4 + }; + AssertRoundtrip(message); + } + + [Test] + public void SingularWrappers_ExplicitNulls() + { + var message = new TestWellKnownTypes(); + var json = new JsonFormatter(new JsonFormatter.Settings(true)).Format(message); + var parsed = JsonParser.Default.Parse<TestWellKnownTypes>(json); + Assert.AreEqual(message, parsed); + } + + [Test] + [TestCase(typeof(Int32Value), "32", 32)] + [TestCase(typeof(Int64Value), "32", 32L)] + [TestCase(typeof(UInt32Value), "32", 32U)] + [TestCase(typeof(UInt64Value), "32", 32UL)] + [TestCase(typeof(StringValue), "\"foo\"", "foo")] + [TestCase(typeof(FloatValue), "1.5", 1.5f)] + [TestCase(typeof(DoubleValue), "1.5", 1.5d)] + public void Wrappers_Standalone(System.Type wrapperType, string json, object expectedValue) + { + IMessage parsed = (IMessage) Activator.CreateInstance(wrapperType); + IMessage expected = (IMessage) Activator.CreateInstance(wrapperType); + JsonParser.Default.Merge(parsed, "null"); + Assert.AreEqual(expected, parsed); + + JsonParser.Default.Merge(parsed, json); + expected.Descriptor.Fields[Wrappers.WrapperValueFieldNumber].Accessor.SetValue(expected, expectedValue); + Assert.AreEqual(expected, parsed); + } + + [Test] + public void BytesWrapper_Standalone() + { + ByteString data = ByteString.CopyFrom(1, 2, 3); + // Can't do this with attributes... + var parsed = JsonParser.Default.Parse<BytesValue>("\"" + data.ToBase64() + "\""); + var expected = new BytesValue { Value = data }; + Assert.AreEqual(expected, parsed); + } + + [Test] + public void RepeatedWrappers() + { + var message = new RepeatedWellKnownTypes + { + BoolField = { true, false }, + BytesField = { ByteString.CopyFrom(1, 2, 3), ByteString.CopyFrom(4, 5, 6), ByteString.Empty }, + DoubleField = { 12.5, -1.5, 0d }, + FloatField = { 123.25f, -20f, 0f }, + Int32Field = { int.MaxValue, int.MinValue, 0 }, + Int64Field = { long.MaxValue, long.MinValue, 0L }, + StringField = { "First", "Second", "" }, + Uint32Field = { uint.MaxValue, uint.MinValue, 0U }, + Uint64Field = { ulong.MaxValue, ulong.MinValue, 0UL }, + }; + AssertRoundtrip(message); + } + + [Test] + public void IndividualWrapperTypes() + { + Assert.AreEqual(new StringValue { Value = "foo" }, StringValue.Parser.ParseJson("\"foo\"")); + Assert.AreEqual(new Int32Value { Value = 1 }, Int32Value.Parser.ParseJson("1")); + // Can parse strings directly too + Assert.AreEqual(new Int32Value { Value = 1 }, Int32Value.Parser.ParseJson("\"1\"")); + } + + private static void AssertRoundtrip<T>(T message) where T : IMessage<T>, new() + { + var clone = message.Clone(); + var json = message.ToString(); + var parsed = JsonParser.Default.Parse<T>(json); + Assert.AreEqual(clone, parsed); + } + + [Test] + [TestCase("0", 0)] + [TestCase("-0", 0)] // Not entirely clear whether we intend to allow this... + [TestCase("1", 1)] + [TestCase("-1", -1)] + [TestCase("2147483647", 2147483647)] + [TestCase("-2147483648", -2147483648)] + public void StringToInt32_Valid(string jsonValue, int expectedParsedValue) + { + string json = "{ \"singleInt32\": \"" + jsonValue + "\"}"; + var parsed = TestAllTypes.Parser.ParseJson(json); + Assert.AreEqual(expectedParsedValue, parsed.SingleInt32); + } + + [Test] + [TestCase("+0")] + [TestCase("00")] + [TestCase("-00")] + [TestCase("--1")] + [TestCase("+1")] + [TestCase("1.5")] + [TestCase("1e10")] + [TestCase("2147483648")] + [TestCase("-2147483649")] + public void StringToInt32_Invalid(string jsonValue) + { + string json = "{ \"singleInt32\": \"" + jsonValue + "\"}"; + Assert.Throws<InvalidProtocolBufferException>(() => TestAllTypes.Parser.ParseJson(json)); + } + + [Test] + [TestCase("0", 0U)] + [TestCase("1", 1U)] + [TestCase("4294967295", 4294967295U)] + public void StringToUInt32_Valid(string jsonValue, uint expectedParsedValue) + { + string json = "{ \"singleUint32\": \"" + jsonValue + "\"}"; + var parsed = TestAllTypes.Parser.ParseJson(json); + Assert.AreEqual(expectedParsedValue, parsed.SingleUint32); + } + + // Assume that anything non-bounds-related is covered in the Int32 case + [Test] + [TestCase("-1")] + [TestCase("4294967296")] + public void StringToUInt32_Invalid(string jsonValue) + { + string json = "{ \"singleUint32\": \"" + jsonValue + "\"}"; + Assert.Throws<InvalidProtocolBufferException>(() => TestAllTypes.Parser.ParseJson(json)); + } + + [Test] + [TestCase("0", 0L)] + [TestCase("1", 1L)] + [TestCase("-1", -1L)] + [TestCase("9223372036854775807", 9223372036854775807)] + [TestCase("-9223372036854775808", -9223372036854775808)] + public void StringToInt64_Valid(string jsonValue, long expectedParsedValue) + { + string json = "{ \"singleInt64\": \"" + jsonValue + "\"}"; + var parsed = TestAllTypes.Parser.ParseJson(json); + Assert.AreEqual(expectedParsedValue, parsed.SingleInt64); + } + + // Assume that anything non-bounds-related is covered in the Int32 case + [Test] + [TestCase("-9223372036854775809")] + [TestCase("9223372036854775808")] + public void StringToInt64_Invalid(string jsonValue) + { + string json = "{ \"singleInt64\": \"" + jsonValue + "\"}"; + Assert.Throws<InvalidProtocolBufferException>(() => TestAllTypes.Parser.ParseJson(json)); + } + + [Test] + [TestCase("0", 0UL)] + [TestCase("1", 1UL)] + [TestCase("18446744073709551615", 18446744073709551615)] + public void StringToUInt64_Valid(string jsonValue, ulong expectedParsedValue) + { + string json = "{ \"singleUint64\": \"" + jsonValue + "\"}"; + var parsed = TestAllTypes.Parser.ParseJson(json); + Assert.AreEqual(expectedParsedValue, parsed.SingleUint64); + } + + // Assume that anything non-bounds-related is covered in the Int32 case + [Test] + [TestCase("-1")] + [TestCase("18446744073709551616")] + public void StringToUInt64_Invalid(string jsonValue) + { + string json = "{ \"singleUint64\": \"" + jsonValue + "\"}"; + Assert.Throws<InvalidProtocolBufferException>(() => TestAllTypes.Parser.ParseJson(json)); + } + + [Test] + [TestCase("0", 0d)] + [TestCase("1", 1d)] + [TestCase("1.000000", 1d)] + [TestCase("1.0000000000000000000000001", 1d)] // We don't notice that we haven't preserved the exact value + [TestCase("-1", -1d)] + [TestCase("1e1", 10d)] + [TestCase("1e01", 10d)] // Leading decimals are allowed in exponents + [TestCase("1E1", 10d)] // Either case is fine + [TestCase("-1e1", -10d)] + [TestCase("1.5e1", 15d)] + [TestCase("-1.5e1", -15d)] + [TestCase("15e-1", 1.5d)] + [TestCase("-15e-1", -1.5d)] + [TestCase("1.79769e308", 1.79769e308)] + [TestCase("-1.79769e308", -1.79769e308)] + [TestCase("Infinity", double.PositiveInfinity)] + [TestCase("-Infinity", double.NegativeInfinity)] + [TestCase("NaN", double.NaN)] + public void StringToDouble_Valid(string jsonValue, double expectedParsedValue) + { + string json = "{ \"singleDouble\": \"" + jsonValue + "\"}"; + var parsed = TestAllTypes.Parser.ParseJson(json); + Assert.AreEqual(expectedParsedValue, parsed.SingleDouble); + } + + [Test] + [TestCase("1.7977e308")] + [TestCase("-1.7977e308")] + [TestCase("1e309")] + [TestCase("1,0")] + [TestCase("1.0.0")] + [TestCase("+1")] + [TestCase("00")] + [TestCase("--1")] + [TestCase("\u00BD")] // 1/2 as a single Unicode character. Just sanity checking... + public void StringToDouble_Invalid(string jsonValue) + { + string json = "{ \"singleDouble\": \"" + jsonValue + "\"}"; + Assert.Throws<InvalidProtocolBufferException>(() => TestAllTypes.Parser.ParseJson(json)); + } + + [Test] + [TestCase("0", 0f)] + [TestCase("1", 1f)] + [TestCase("1.000000", 1f)] + [TestCase("-1", -1f)] + [TestCase("3.402823e38", 3.402823e38f)] + [TestCase("-3.402823e38", -3.402823e38f)] + [TestCase("1.5e1", 15f)] + [TestCase("15e-1", 1.5f)] + public void StringToFloat_Valid(string jsonValue, float expectedParsedValue) + { + string json = "{ \"singleFloat\": \"" + jsonValue + "\"}"; + var parsed = TestAllTypes.Parser.ParseJson(json); + Assert.AreEqual(expectedParsedValue, parsed.SingleFloat); + } + + [Test] + [TestCase("3.402824e38")] + [TestCase("-3.402824e38")] + [TestCase("1,0")] + [TestCase("1.0.0")] + [TestCase("+1")] + [TestCase("00")] + [TestCase("--1")] + public void StringToFloat_Invalid(string jsonValue) + { + string json = "{ \"singleFloat\": \"" + jsonValue + "\"}"; + Assert.Throws<InvalidProtocolBufferException>(() => TestAllTypes.Parser.ParseJson(json)); + } + + [Test] + [TestCase("0", 0)] + [TestCase("-0", 0)] // Not entirely clear whether we intend to allow this... + [TestCase("1", 1)] + [TestCase("-1", -1)] + [TestCase("2147483647", 2147483647)] + [TestCase("-2147483648", -2147483648)] + public void NumberToInt32_Valid(string jsonValue, int expectedParsedValue) + { + string json = "{ \"singleInt32\": " + jsonValue + "}"; + var parsed = TestAllTypes.Parser.ParseJson(json); + Assert.AreEqual(expectedParsedValue, parsed.SingleInt32); + } + + [Test] + [TestCase("+0", typeof(InvalidJsonException))] + [TestCase("00", typeof(InvalidJsonException))] + [TestCase("-00", typeof(InvalidJsonException))] + [TestCase("--1", typeof(InvalidJsonException))] + [TestCase("+1", typeof(InvalidJsonException))] + [TestCase("1.5", typeof(InvalidProtocolBufferException), Ignore = true, Reason = "Desired behaviour unclear")] + [TestCase("1e10", typeof(InvalidProtocolBufferException))] + [TestCase("2147483648", typeof(InvalidProtocolBufferException))] + [TestCase("-2147483649", typeof(InvalidProtocolBufferException))] + public void NumberToInt32_Invalid(string jsonValue, System.Type expectedExceptionType) + { + string json = "{ \"singleInt32\": " + jsonValue + "}"; + Assert.Throws(expectedExceptionType, () => TestAllTypes.Parser.ParseJson(json)); + } + + [Test] + [TestCase("0", 0U)] + [TestCase("1", 1U)] + [TestCase("4294967295", 4294967295U)] + public void NumberToUInt32_Valid(string jsonValue, uint expectedParsedValue) + { + string json = "{ \"singleUint32\": " + jsonValue + "}"; + var parsed = TestAllTypes.Parser.ParseJson(json); + Assert.AreEqual(expectedParsedValue, parsed.SingleUint32); + } + + // Assume that anything non-bounds-related is covered in the Int32 case + [Test] + [TestCase("-1")] + [TestCase("4294967296")] + public void NumberToUInt32_Invalid(string jsonValue) + { + string json = "{ \"singleUint32\": " + jsonValue + "}"; + Assert.Throws<InvalidProtocolBufferException>(() => TestAllTypes.Parser.ParseJson(json)); + } + + [Test] + [TestCase("0", 0L)] + [TestCase("1", 1L)] + [TestCase("-1", -1L)] + [TestCase("9223372036854775807", 9223372036854775807, Ignore = true, Reason = "Desired behaviour unclear")] + [TestCase("-9223372036854775808", -9223372036854775808, Ignore = true, Reason = "Desired behaviour unclear")] + public void NumberToInt64_Valid(string jsonValue, long expectedParsedValue) + { + string json = "{ \"singleInt64\": " + jsonValue + "}"; + var parsed = TestAllTypes.Parser.ParseJson(json); + Assert.AreEqual(expectedParsedValue, parsed.SingleInt64); + } + + // Assume that anything non-bounds-related is covered in the Int32 case + [Test] + [TestCase("-9223372036854775809", Ignore = true, Reason = "Desired behaviour unclear")] + [TestCase("9223372036854775808", Ignore = true, Reason = "Desired behaviour unclear")] + public void NumberToInt64_Invalid(string jsonValue) + { + string json = "{ \"singleInt64\": " + jsonValue + "}"; + Assert.Throws<InvalidProtocolBufferException>(() => TestAllTypes.Parser.ParseJson(json)); + } + + [Test] + [TestCase("0", 0UL)] + [TestCase("1", 1UL)] + [TestCase("18446744073709551615", 18446744073709551615, Ignore = true, Reason = "Desired behaviour unclear")] + public void NumberToUInt64_Valid(string jsonValue, ulong expectedParsedValue) + { + string json = "{ \"singleUint64\": " + jsonValue + "}"; + var parsed = TestAllTypes.Parser.ParseJson(json); + Assert.AreEqual(expectedParsedValue, parsed.SingleUint64); + } + + // Assume that anything non-bounds-related is covered in the Int32 case + [Test] + [TestCase("-1")] + [TestCase("18446744073709551616")] + public void NumberToUInt64_Invalid(string jsonValue) + { + string json = "{ \"singleUint64\": " + jsonValue + "}"; + Assert.Throws<InvalidProtocolBufferException>(() => TestAllTypes.Parser.ParseJson(json)); + } + + [Test] + [TestCase("0", 0d)] + [TestCase("1", 1d)] + [TestCase("1.000000", 1d)] + [TestCase("1.0000000000000000000000001", 1d)] // We don't notice that we haven't preserved the exact value + [TestCase("-1", -1d)] + [TestCase("1e1", 10d)] + [TestCase("1e01", 10d)] // Leading decimals are allowed in exponents + [TestCase("1E1", 10d)] // Either case is fine + [TestCase("-1e1", -10d)] + [TestCase("1.5e1", 15d)] + [TestCase("-1.5e1", -15d)] + [TestCase("15e-1", 1.5d)] + [TestCase("-15e-1", -1.5d)] + [TestCase("1.79769e308", 1.79769e308)] + [TestCase("-1.79769e308", -1.79769e308)] + public void NumberToDouble_Valid(string jsonValue, double expectedParsedValue) + { + string json = "{ \"singleDouble\": " + jsonValue + "}"; + var parsed = TestAllTypes.Parser.ParseJson(json); + Assert.AreEqual(expectedParsedValue, parsed.SingleDouble); + } + + [Test] + [TestCase("1.7977e308", Ignore = true, Reason = "Desired behaviour unclear")] + [TestCase("-1.7977e308", Ignore = true, Reason = "Desired behaviour unclear")] + [TestCase("1e309", Ignore = true, Reason = "Desired behaviour unclear")] + [TestCase("1,0")] + [TestCase("1.0.0")] + [TestCase("+1")] + [TestCase("00")] + [TestCase("--1")] + [TestCase("\u00BD")] // 1/2 as a single Unicode character. Just sanity checking... + public void NumberToDouble_Invalid(string jsonValue) + { + string json = "{ \"singleDouble\": " + jsonValue + "}"; + Assert.Throws<InvalidJsonException>(() => TestAllTypes.Parser.ParseJson(json)); + } + + [Test] + [TestCase("0", 0f)] + [TestCase("1", 1f)] + [TestCase("1.000000", 1f)] + [TestCase("-1", -1f)] + [TestCase("3.402823e38", 3.402823e38f)] + [TestCase("-3.402823e38", -3.402823e38f)] + [TestCase("1.5e1", 15f)] + [TestCase("15e-1", 1.5f)] + public void NumberToFloat_Valid(string jsonValue, float expectedParsedValue) + { + string json = "{ \"singleFloat\": " + jsonValue + "}"; + var parsed = TestAllTypes.Parser.ParseJson(json); + Assert.AreEqual(expectedParsedValue, parsed.SingleFloat); + } + + [Test] + [TestCase("3.402824e38", typeof(InvalidProtocolBufferException))] + [TestCase("-3.402824e38", typeof(InvalidProtocolBufferException))] + [TestCase("1,0", typeof(InvalidJsonException))] + [TestCase("1.0.0", typeof(InvalidJsonException))] + [TestCase("+1", typeof(InvalidJsonException))] + [TestCase("00", typeof(InvalidJsonException))] + [TestCase("--1", typeof(InvalidJsonException))] + public void NumberToFloat_Invalid(string jsonValue, System.Type expectedExceptionType) + { + string json = "{ \"singleFloat\": " + jsonValue + "}"; + Assert.Throws(expectedExceptionType, () => TestAllTypes.Parser.ParseJson(json)); + } + + // The simplest way of testing that the value has parsed correctly is to reformat it, + // as we trust the formatting. In many cases that will give the same result as the input, + // so in those cases we accept an expectedFormatted value of null. Sometimes the results + // will be different though, due to a different number of digits being provided. + [Test] + // Z offset + [TestCase("2015-10-09T14:46:23.123456789Z", null)] + [TestCase("2015-10-09T14:46:23.123456Z", null)] + [TestCase("2015-10-09T14:46:23.123Z", null)] + [TestCase("2015-10-09T14:46:23Z", null)] + [TestCase("2015-10-09T14:46:23.123456000Z", "2015-10-09T14:46:23.123456Z")] + [TestCase("2015-10-09T14:46:23.1234560Z", "2015-10-09T14:46:23.123456Z")] + [TestCase("2015-10-09T14:46:23.123000000Z", "2015-10-09T14:46:23.123Z")] + [TestCase("2015-10-09T14:46:23.1230Z", "2015-10-09T14:46:23.123Z")] + [TestCase("2015-10-09T14:46:23.00Z", "2015-10-09T14:46:23Z")] + + // +00:00 offset + [TestCase("2015-10-09T14:46:23.123456789+00:00", "2015-10-09T14:46:23.123456789Z")] + [TestCase("2015-10-09T14:46:23.123456+00:00", "2015-10-09T14:46:23.123456Z")] + [TestCase("2015-10-09T14:46:23.123+00:00", "2015-10-09T14:46:23.123Z")] + [TestCase("2015-10-09T14:46:23+00:00", "2015-10-09T14:46:23Z")] + [TestCase("2015-10-09T14:46:23.123456000+00:00", "2015-10-09T14:46:23.123456Z")] + [TestCase("2015-10-09T14:46:23.1234560+00:00", "2015-10-09T14:46:23.123456Z")] + [TestCase("2015-10-09T14:46:23.123000000+00:00", "2015-10-09T14:46:23.123Z")] + [TestCase("2015-10-09T14:46:23.1230+00:00", "2015-10-09T14:46:23.123Z")] + [TestCase("2015-10-09T14:46:23.00+00:00", "2015-10-09T14:46:23Z")] + + // Other offsets (assume by now that the subsecond handling is okay) + [TestCase("2015-10-09T15:46:23.123456789+01:00", "2015-10-09T14:46:23.123456789Z")] + [TestCase("2015-10-09T13:46:23.123456789-01:00", "2015-10-09T14:46:23.123456789Z")] + [TestCase("2015-10-09T15:16:23.123456789+00:30", "2015-10-09T14:46:23.123456789Z")] + [TestCase("2015-10-09T14:16:23.123456789-00:30", "2015-10-09T14:46:23.123456789Z")] + [TestCase("2015-10-09T16:31:23.123456789+01:45", "2015-10-09T14:46:23.123456789Z")] + [TestCase("2015-10-09T13:01:23.123456789-01:45", "2015-10-09T14:46:23.123456789Z")] + [TestCase("2015-10-10T08:46:23.123456789+18:00", "2015-10-09T14:46:23.123456789Z")] + [TestCase("2015-10-08T20:46:23.123456789-18:00", "2015-10-09T14:46:23.123456789Z")] + + // Leap years and min/max + [TestCase("2016-02-29T14:46:23.123456789Z", null)] + [TestCase("2000-02-29T14:46:23.123456789Z", null)] + [TestCase("0001-01-01T00:00:00Z", null)] + [TestCase("9999-12-31T23:59:59.999999999Z", null)] + public void Timestamp_Valid(string jsonValue, string expectedFormatted) + { + expectedFormatted = expectedFormatted ?? jsonValue; + string json = "\"" + jsonValue + "\""; + var parsed = Timestamp.Parser.ParseJson(json); + Assert.AreEqual(expectedFormatted, parsed.ToString()); + } + + [Test] + [TestCase("2015-10-09 14:46:23.123456789Z", Description = "No T between date and time")] + [TestCase("2015/10/09T14:46:23.123456789Z", Description = "Wrong date separators")] + [TestCase("2015-10-09T14.46.23.123456789Z", Description = "Wrong time separators")] + [TestCase("2015-10-09T14:46:23,123456789Z", Description = "Wrong fractional second separators (valid ISO-8601 though)")] + [TestCase(" 2015-10-09T14:46:23.123456789Z", Description = "Whitespace at start")] + [TestCase("2015-10-09T14:46:23.123456789Z ", Description = "Whitespace at end")] + [TestCase("2015-10-09T14:46:23.1234567890", Description = "Too many digits")] + [TestCase("2015-10-09T14:46:23.123456789", Description = "No offset")] + [TestCase("2015-13-09T14:46:23.123456789Z", Description = "Invalid month")] + [TestCase("2015-10-32T14:46:23.123456789Z", Description = "Invalid day")] + [TestCase("2015-10-09T24:00:00.000000000Z", Description = "Invalid hour (valid ISO-8601 though)")] + [TestCase("2015-10-09T14:60:23.123456789Z", Description = "Invalid minutes")] + [TestCase("2015-10-09T14:46:60.123456789Z", Description = "Invalid seconds")] + [TestCase("2015-10-09T14:46:23.123456789+18:01", Description = "Offset too large (positive)")] + [TestCase("2015-10-09T14:46:23.123456789-18:01", Description = "Offset too large (negative)")] + [TestCase("2015-10-09T14:46:23.123456789-00:00", Description = "Local offset (-00:00) makes no sense here")] + [TestCase("0001-01-01T00:00:00+00:01", Description = "Value before earliest when offset applied")] + [TestCase("9999-12-31T23:59:59.999999999-00:01", Description = "Value after latest when offset applied")] + [TestCase("2100-02-29T14:46:23.123456789Z", Description = "Feb 29th on a non-leap-year")] + public void Timestamp_Invalid(string jsonValue) + { + string json = "\"" + jsonValue + "\""; + Assert.Throws<InvalidProtocolBufferException>(() => Timestamp.Parser.ParseJson(json)); + } + + [Test] + public void StructValue_Null() + { + Assert.AreEqual(new Value { NullValue = 0 }, Value.Parser.ParseJson("null")); + } + + [Test] + public void StructValue_String() + { + Assert.AreEqual(new Value { StringValue = "hi" }, Value.Parser.ParseJson("\"hi\"")); + } + + [Test] + public void StructValue_Bool() + { + Assert.AreEqual(new Value { BoolValue = true }, Value.Parser.ParseJson("true")); + Assert.AreEqual(new Value { BoolValue = false }, Value.Parser.ParseJson("false")); + } + + [Test] + public void StructValue_List() + { + Assert.AreEqual(Value.ForList(Value.ForNumber(1), Value.ForString("x")), Value.Parser.ParseJson("[1, \"x\"]")); + } + + [Test] + public void ParseListValue() + { + Assert.AreEqual(new ListValue { Values = { Value.ForNumber(1), Value.ForString("x") } }, ListValue.Parser.ParseJson("[1, \"x\"]")); + } + + [Test] + public void StructValue_Struct() + { + Assert.AreEqual( + Value.ForStruct(new Struct { Fields = { { "x", Value.ForNumber(1) }, { "y", Value.ForString("z") } } }), + Value.Parser.ParseJson("{ \"x\": 1, \"y\": \"z\" }")); + } + + [Test] + public void ParseStruct() + { + Assert.AreEqual(new Struct { Fields = { { "x", Value.ForNumber(1) }, { "y", Value.ForString("z") } } }, + Struct.Parser.ParseJson("{ \"x\": 1, \"y\": \"z\" }")); + } + + // TODO for duration parsing: upper and lower bounds. + // +/- 315576000000 seconds + + [Test] + [TestCase("1.123456789s", null)] + [TestCase("1.123456s", null)] + [TestCase("1.123s", null)] + [TestCase("1.12300s", "1.123s")] + [TestCase("1.12345s", "1.123450s")] + [TestCase("1s", null)] + [TestCase("-1.123456789s", null)] + [TestCase("-1.123456s", null)] + [TestCase("-1.123s", null)] + [TestCase("-1s", null)] + [TestCase("0.123s", null)] + [TestCase("-0.123s", null)] + [TestCase("123456.123s", null)] + [TestCase("-123456.123s", null)] + // Upper and lower bounds + [TestCase("315576000000s", null)] + [TestCase("-315576000000s", null)] + public void Duration_Valid(string jsonValue, string expectedFormatted) + { + expectedFormatted = expectedFormatted ?? jsonValue; + string json = "\"" + jsonValue + "\""; + var parsed = Duration.Parser.ParseJson(json); + Assert.AreEqual(expectedFormatted, parsed.ToString()); + } + + // The simplest way of testing that the value has parsed correctly is to reformat it, + // as we trust the formatting. In many cases that will give the same result as the input, + // so in those cases we accept an expectedFormatted value of null. Sometimes the results + // will be different though, due to a different number of digits being provided. + [Test] + [TestCase("1.1234567890s", Description = "Too many digits")] + [TestCase("1.123456789", Description = "No suffix")] + [TestCase("1.123456789ss", Description = "Too much suffix")] + [TestCase("1.123456789S", Description = "Upper case suffix")] + [TestCase("+1.123456789s", Description = "Leading +")] + [TestCase(".123456789s", Description = "No integer before the fraction")] + [TestCase("1,123456789s", Description = "Comma as decimal separator")] + [TestCase("1x1.123456789s", Description = "Non-digit in integer part")] + [TestCase("1.1x3456789s", Description = "Non-digit in fractional part")] + [TestCase(" 1.123456789s", Description = "Whitespace before fraction")] + [TestCase("1.123456789s ", Description = "Whitespace after value")] + [TestCase("01.123456789s", Description = "Leading zero (positive)")] + [TestCase("-01.123456789s", Description = "Leading zero (negative)")] + [TestCase("--0.123456789s", Description = "Double minus sign")] + // Violate upper/lower bounds in various ways + [TestCase("315576000001s", Description = "Integer part too large")] + [TestCase("315576000000.000000001s", Description = "Integer part is upper bound; non-zero fraction")] + [TestCase("3155760000000s", Description = "Integer part too long (positive)")] + [TestCase("-3155760000000s", Description = "Integer part too long (negative)")] + public void Duration_Invalid(string jsonValue) + { + string json = "\"" + jsonValue + "\""; + Assert.Throws<InvalidProtocolBufferException>(() => Duration.Parser.ParseJson(json)); + } + + // Not as many tests for field masks as I'd like; more to be added when we have more + // detailed specifications. + + [Test] + [TestCase("")] + [TestCase("foo", "foo")] + [TestCase("foo,bar", "foo", "bar")] + [TestCase("foo.bar", "foo.bar")] + [TestCase("fooBar", "foo_bar")] + [TestCase("fooBar.bazQux", "foo_bar.baz_qux")] + public void FieldMask_Valid(string jsonValue, params string[] expectedPaths) + { + string json = "\"" + jsonValue + "\""; + var parsed = FieldMask.Parser.ParseJson(json); + CollectionAssert.AreEqual(expectedPaths, parsed.Paths); + } + + [Test] + public void DataAfterObject() + { + string json = "{} 10"; + Assert.Throws<InvalidJsonException>(() => TestAllTypes.Parser.ParseJson(json)); + } + + /// <summary> + /// JSON equivalent to <see cref="CodedInputStreamTest.MaliciousRecursion"/> + /// </summary> + [Test] + public void MaliciousRecursion() + { + string data64 = CodedInputStreamTest.MakeRecursiveMessage(64).ToString(); + string data65 = CodedInputStreamTest.MakeRecursiveMessage(65).ToString(); + + var parser64 = new JsonParser(new JsonParser.Settings(64)); + CodedInputStreamTest.AssertMessageDepth(parser64.Parse<TestRecursiveMessage>(data64), 64); + Assert.Throws<InvalidProtocolBufferException>(() => parser64.Parse<TestRecursiveMessage>(data65)); + + var parser63 = new JsonParser(new JsonParser.Settings(63)); + Assert.Throws<InvalidProtocolBufferException>(() => parser63.Parse<TestRecursiveMessage>(data64)); + + } + } +} diff --git a/csharp/src/Google.Protobuf.Test/JsonTokenizerTest.cs b/csharp/src/Google.Protobuf.Test/JsonTokenizerTest.cs new file mode 100644 index 00000000..a38efeed --- /dev/null +++ b/csharp/src/Google.Protobuf.Test/JsonTokenizerTest.cs @@ -0,0 +1,409 @@ +#region Copyright notice and license +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#endregion +using NUnit.Framework; +using System; +using System.IO; + +namespace Google.Protobuf +{ + public class JsonTokenizerTest + { + [Test] + public void EmptyObjectValue() + { + AssertTokens("{}", JsonToken.StartObject, JsonToken.EndObject); + } + + [Test] + public void EmptyArrayValue() + { + AssertTokens("[]", JsonToken.StartArray, JsonToken.EndArray); + } + + [Test] + [TestCase("foo", "foo")] + [TestCase("tab\\t", "tab\t")] + [TestCase("line\\nfeed", "line\nfeed")] + [TestCase("carriage\\rreturn", "carriage\rreturn")] + [TestCase("back\\bspace", "back\bspace")] + [TestCase("form\\ffeed", "form\ffeed")] + [TestCase("escaped\\/slash", "escaped/slash")] + [TestCase("escaped\\\\backslash", "escaped\\backslash")] + [TestCase("escaped\\\"quote", "escaped\"quote")] + [TestCase("foo {}[] bar", "foo {}[] bar")] + [TestCase("foo\\u09aFbar", "foo\u09afbar")] // Digits, upper hex, lower hex + [TestCase("ab\ud800\udc00cd", "ab\ud800\udc00cd")] + [TestCase("ab\\ud800\\udc00cd", "ab\ud800\udc00cd")] + public void StringValue(string json, string expectedValue) + { + AssertTokensNoReplacement("\"" + json + "\"", JsonToken.Value(expectedValue)); + } + + // Valid surrogate pairs, with mixed escaping. These test cases can't be expressed + // using TestCase as they have no valid UTF-8 representation. + // It's unclear exactly how we should handle a mixture of escaped or not: that can't + // come from UTF-8 text, but could come from a .NET string. For the moment, + // treat it as valid in the obvious way. + [Test] + public void MixedSurrogatePairs() + { + string expected = "\ud800\udc00"; + AssertTokens("'\\ud800\udc00'", JsonToken.Value(expected)); + AssertTokens("'\ud800\\udc00'", JsonToken.Value(expected)); + } + + [Test] + public void ObjectDepth() + { + string json = "{ \"foo\": { \"x\": 1, \"y\": [ 0 ] } }"; + var tokenizer = new JsonTokenizer(new StringReader(json)); + // If we had more tests like this, I'd introduce a helper method... but for one test, it's not worth it. + Assert.AreEqual(0, tokenizer.ObjectDepth); + Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); + Assert.AreEqual(1, tokenizer.ObjectDepth); + Assert.AreEqual(JsonToken.Name("foo"), tokenizer.Next()); + Assert.AreEqual(1, tokenizer.ObjectDepth); + Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); + Assert.AreEqual(2, tokenizer.ObjectDepth); + Assert.AreEqual(JsonToken.Name("x"), tokenizer.Next()); + Assert.AreEqual(2, tokenizer.ObjectDepth); + Assert.AreEqual(JsonToken.Value(1), tokenizer.Next()); + Assert.AreEqual(2, tokenizer.ObjectDepth); + Assert.AreEqual(JsonToken.Name("y"), tokenizer.Next()); + Assert.AreEqual(2, tokenizer.ObjectDepth); + Assert.AreEqual(JsonToken.StartArray, tokenizer.Next()); + Assert.AreEqual(2, tokenizer.ObjectDepth); // Depth hasn't changed in array + Assert.AreEqual(JsonToken.Value(0), tokenizer.Next()); + Assert.AreEqual(2, tokenizer.ObjectDepth); + Assert.AreEqual(JsonToken.EndArray, tokenizer.Next()); + Assert.AreEqual(2, tokenizer.ObjectDepth); + Assert.AreEqual(JsonToken.EndObject, tokenizer.Next()); + Assert.AreEqual(1, tokenizer.ObjectDepth); + Assert.AreEqual(JsonToken.EndObject, tokenizer.Next()); + Assert.AreEqual(0, tokenizer.ObjectDepth); + Assert.AreEqual(JsonToken.EndDocument, tokenizer.Next()); + Assert.AreEqual(0, tokenizer.ObjectDepth); + } + + [Test] + public void ObjectDepth_WithPushBack() + { + string json = "{}"; + var tokenizer = new JsonTokenizer(new StringReader(json)); + Assert.AreEqual(0, tokenizer.ObjectDepth); + var token = tokenizer.Next(); + Assert.AreEqual(1, tokenizer.ObjectDepth); + // When we push back a "start object", we should effectively be back to the previous depth. + tokenizer.PushBack(token); + Assert.AreEqual(0, tokenizer.ObjectDepth); + // Read the same token again, and get back to depth 1 + token = tokenizer.Next(); + Assert.AreEqual(1, tokenizer.ObjectDepth); + + // Now the same in reverse, with EndObject + token = tokenizer.Next(); + Assert.AreEqual(0, tokenizer.ObjectDepth); + tokenizer.PushBack(token); + Assert.AreEqual(1, tokenizer.ObjectDepth); + tokenizer.Next(); + Assert.AreEqual(0, tokenizer.ObjectDepth); + } + + [Test] + [TestCase("embedded tab\t")] + [TestCase("embedded CR\r")] + [TestCase("embedded LF\n")] + [TestCase("embedded bell\u0007")] + [TestCase("bad escape\\a")] + [TestCase("incomplete escape\\")] + [TestCase("incomplete Unicode escape\\u000")] + [TestCase("invalid Unicode escape\\u000H")] + // Surrogate pair handling, both in raw .NET strings and escaped. We only need + // to detect this in strings, as non-ASCII characters anywhere other than in strings + // will already lead to parsing errors. + [TestCase("\\ud800")] + [TestCase("\\udc00")] + [TestCase("\\ud800x")] + [TestCase("\\udc00x")] + [TestCase("\\udc00\\ud800y")] + public void InvalidStringValue(string json) + { + AssertThrowsAfter("\"" + json + "\""); + } + + // Tests for invalid strings that can't be expressed in attributes, + // as the constants can't be expressed as UTF-8 strings. + [Test] + public void InvalidSurrogatePairs() + { + AssertThrowsAfter("\"\ud800x\""); + AssertThrowsAfter("\"\udc00y\""); + AssertThrowsAfter("\"\udc00\ud800y\""); + } + + [Test] + [TestCase("0", 0)] + [TestCase("-0", 0)] // We don't distinguish between positive and negative 0 + [TestCase("1", 1)] + [TestCase("-1", -1)] + // From here on, assume leading sign is okay... + [TestCase("1.125", 1.125)] + [TestCase("1.0", 1)] + [TestCase("1e5", 100000)] + [TestCase("1e000000", 1)] // Weird, but not prohibited by the spec + [TestCase("1E5", 100000)] + [TestCase("1e+5", 100000)] + [TestCase("1E-5", 0.00001)] + [TestCase("123E-2", 1.23)] + [TestCase("123.45E3", 123450)] + [TestCase(" 1 ", 1)] + public void NumberValue(string json, double expectedValue) + { + AssertTokens(json, JsonToken.Value(expectedValue)); + } + + [Test] + [TestCase("00")] + [TestCase(".5")] + [TestCase("1.")] + [TestCase("1e")] + [TestCase("1e-")] + [TestCase("--")] + [TestCase("--1")] + [TestCase("-1.7977e308")] + [TestCase("1.7977e308")] + public void InvalidNumberValue(string json) + { + AssertThrowsAfter(json); + } + + [Test] + [TestCase("nul")] + [TestCase("nothing")] + [TestCase("truth")] + [TestCase("fALSEhood")] + public void InvalidLiterals(string json) + { + AssertThrowsAfter(json); + } + + [Test] + public void NullValue() + { + AssertTokens("null", JsonToken.Null); + } + + [Test] + public void TrueValue() + { + AssertTokens("true", JsonToken.True); + } + + [Test] + public void FalseValue() + { + AssertTokens("false", JsonToken.False); + } + + [Test] + public void SimpleObject() + { + AssertTokens("{'x': 'y'}", + JsonToken.StartObject, JsonToken.Name("x"), JsonToken.Value("y"), JsonToken.EndObject); + } + + [Test] + [TestCase("[10, 20", 3)] + [TestCase("[10,", 2)] + [TestCase("[10:20]", 2)] + [TestCase("[", 1)] + [TestCase("[,", 1)] + [TestCase("{", 1)] + [TestCase("{,", 1)] + [TestCase("{", 1)] + [TestCase("{[", 1)] + [TestCase("{{", 1)] + [TestCase("{0", 1)] + [TestCase("{null", 1)] + [TestCase("{false", 1)] + [TestCase("{true", 1)] + [TestCase("}", 0)] + [TestCase("]", 0)] + [TestCase(",", 0)] + [TestCase("'foo' 'bar'", 1)] + [TestCase(":", 0)] + [TestCase("'foo", 0)] // Incomplete string + [TestCase("{ 'foo' }", 2)] + [TestCase("{ x:1", 1)] // Property names must be quoted + [TestCase("{]", 1)] + [TestCase("[}", 1)] + [TestCase("[1,", 2)] + [TestCase("{'x':0]", 3)] + [TestCase("{ 'foo': }", 2)] + [TestCase("{ 'foo':'bar', }", 3)] + public void InvalidStructure(string json, int expectedValidTokens) + { + // Note: we don't test that the earlier tokens are exactly as expected, + // partly because that's hard to parameterize. + var reader = new StringReader(json.Replace('\'', '"')); + var tokenizer = new JsonTokenizer(reader); + for (int i = 0; i < expectedValidTokens; i++) + { + Assert.IsNotNull(tokenizer.Next()); + } + Assert.Throws<InvalidJsonException>(() => tokenizer.Next()); + } + + [Test] + public void ArrayMixedType() + { + AssertTokens("[1, 'foo', null, false, true, [2], {'x':'y' }]", + JsonToken.StartArray, + JsonToken.Value(1), + JsonToken.Value("foo"), + JsonToken.Null, + JsonToken.False, + JsonToken.True, + JsonToken.StartArray, + JsonToken.Value(2), + JsonToken.EndArray, + JsonToken.StartObject, + JsonToken.Name("x"), + JsonToken.Value("y"), + JsonToken.EndObject, + JsonToken.EndArray); + } + + [Test] + public void ObjectMixedType() + { + AssertTokens(@"{'a': 1, 'b': 'bar', 'c': null, 'd': false, 'e': true, + 'f': [2], 'g': {'x':'y' }}", + JsonToken.StartObject, + JsonToken.Name("a"), + JsonToken.Value(1), + JsonToken.Name("b"), + JsonToken.Value("bar"), + JsonToken.Name("c"), + JsonToken.Null, + JsonToken.Name("d"), + JsonToken.False, + JsonToken.Name("e"), + JsonToken.True, + JsonToken.Name("f"), + JsonToken.StartArray, + JsonToken.Value(2), + JsonToken.EndArray, + JsonToken.Name("g"), + JsonToken.StartObject, + JsonToken.Name("x"), + JsonToken.Value("y"), + JsonToken.EndObject, + JsonToken.EndObject); + } + + [Test] + public void NextAfterEndDocumentThrows() + { + var tokenizer = new JsonTokenizer(new StringReader("null")); + Assert.AreEqual(JsonToken.Null, tokenizer.Next()); + Assert.AreEqual(JsonToken.EndDocument, tokenizer.Next()); + Assert.Throws<InvalidOperationException>(() => tokenizer.Next()); + } + + [Test] + public void CanPushBackEndDocument() + { + var tokenizer = new JsonTokenizer(new StringReader("null")); + Assert.AreEqual(JsonToken.Null, tokenizer.Next()); + Assert.AreEqual(JsonToken.EndDocument, tokenizer.Next()); + tokenizer.PushBack(JsonToken.EndDocument); + Assert.AreEqual(JsonToken.EndDocument, tokenizer.Next()); + Assert.Throws<InvalidOperationException>(() => tokenizer.Next()); + } + + /// <summary> + /// Asserts that the specified JSON is tokenized into the given sequence of tokens. + /// All apostrophes are first converted to double quotes, allowing any tests + /// that don't need to check actual apostrophe handling to use apostrophes in the JSON, avoiding + /// messy string literal escaping. The "end document" token is not specified in the list of + /// expected tokens, but is implicit. + /// </summary> + private static void AssertTokens(string json, params JsonToken[] expectedTokens) + { + AssertTokensNoReplacement(json.Replace('\'', '"'), expectedTokens); + } + + /// <summary> + /// Asserts that the specified JSON is tokenized into the given sequence of tokens. + /// Unlike <see cref="AssertTokens(string, JsonToken[])"/>, this does not perform any character + /// replacement on the specified JSON, and should be used when the text contains apostrophes which + /// are expected to be used *as* apostrophes. The "end document" token is not specified in the list of + /// expected tokens, but is implicit. + /// </summary> + private static void AssertTokensNoReplacement(string json, params JsonToken[] expectedTokens) + { + var reader = new StringReader(json); + var tokenizer = new JsonTokenizer(reader); + for (int i = 0; i < expectedTokens.Length; i++) + { + var actualToken = tokenizer.Next(); + if (actualToken == JsonToken.EndDocument) + { + Assert.Fail("Expected {0} but reached end of token stream", expectedTokens[i]); + } + Assert.AreEqual(expectedTokens[i], actualToken); + } + var finalToken = tokenizer.Next(); + if (finalToken != JsonToken.EndDocument) + { + Assert.Fail("Expected token stream to be exhausted; received {0}", finalToken); + } + } + + private static void AssertThrowsAfter(string json, params JsonToken[] expectedTokens) + { + var reader = new StringReader(json); + var tokenizer = new JsonTokenizer(reader); + for (int i = 0; i < expectedTokens.Length; i++) + { + var actualToken = tokenizer.Next(); + if (actualToken == JsonToken.EndDocument) + { + Assert.Fail("Expected {0} but reached end of document", expectedTokens[i]); + } + Assert.AreEqual(expectedTokens[i], actualToken); + } + Assert.Throws<InvalidJsonException>(() => tokenizer.Next()); + } + } +} diff --git a/csharp/src/Google.Protobuf/FieldCodec.cs b/csharp/src/Google.Protobuf/FieldCodec.cs index 20a1f438..54cbc209 100644 --- a/csharp/src/Google.Protobuf/FieldCodec.cs +++ b/csharp/src/Google.Protobuf/FieldCodec.cs @@ -30,6 +30,7 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endregion +using Google.Protobuf.WellKnownTypes; using System; using System.Collections.Generic; @@ -261,20 +262,17 @@ namespace Google.Protobuf /// </remarks> private static class WrapperCodecs { - // All the field numbers are the same (1). - private const int WrapperValueFieldNumber = Google.Protobuf.WellKnownTypes.Int32Value.ValueFieldNumber; - - private static readonly Dictionary<Type, object> Codecs = new Dictionary<Type, object> + private static readonly Dictionary<System.Type, object> Codecs = new Dictionary<System.Type, object> { - { typeof(bool), ForBool(WireFormat.MakeTag(WrapperValueFieldNumber, WireFormat.WireType.Varint)) }, - { typeof(int), ForInt32(WireFormat.MakeTag(WrapperValueFieldNumber, WireFormat.WireType.Varint)) }, - { typeof(long), ForInt64(WireFormat.MakeTag(WrapperValueFieldNumber, WireFormat.WireType.Varint)) }, - { typeof(uint), ForUInt32(WireFormat.MakeTag(WrapperValueFieldNumber, WireFormat.WireType.Varint)) }, - { typeof(ulong), ForUInt64(WireFormat.MakeTag(WrapperValueFieldNumber, WireFormat.WireType.Varint)) }, - { typeof(float), ForFloat(WireFormat.MakeTag(WrapperValueFieldNumber, WireFormat.WireType.Fixed32)) }, - { typeof(double), ForDouble(WireFormat.MakeTag(WrapperValueFieldNumber, WireFormat.WireType.Fixed64)) }, - { typeof(string), ForString(WireFormat.MakeTag(WrapperValueFieldNumber, WireFormat.WireType.LengthDelimited)) }, - { typeof(ByteString), ForBytes(WireFormat.MakeTag(WrapperValueFieldNumber, WireFormat.WireType.LengthDelimited)) } + { typeof(bool), ForBool(WireFormat.MakeTag(Wrappers.WrapperValueFieldNumber, WireFormat.WireType.Varint)) }, + { typeof(int), ForInt32(WireFormat.MakeTag(Wrappers.WrapperValueFieldNumber, WireFormat.WireType.Varint)) }, + { typeof(long), ForInt64(WireFormat.MakeTag(Wrappers.WrapperValueFieldNumber, WireFormat.WireType.Varint)) }, + { typeof(uint), ForUInt32(WireFormat.MakeTag(Wrappers.WrapperValueFieldNumber, WireFormat.WireType.Varint)) }, + { typeof(ulong), ForUInt64(WireFormat.MakeTag(Wrappers.WrapperValueFieldNumber, WireFormat.WireType.Varint)) }, + { typeof(float), ForFloat(WireFormat.MakeTag(Wrappers.WrapperValueFieldNumber, WireFormat.WireType.Fixed32)) }, + { typeof(double), ForDouble(WireFormat.MakeTag(Wrappers.WrapperValueFieldNumber, WireFormat.WireType.Fixed64)) }, + { typeof(string), ForString(WireFormat.MakeTag(Wrappers.WrapperValueFieldNumber, WireFormat.WireType.LengthDelimited)) }, + { typeof(ByteString), ForBytes(WireFormat.MakeTag(Wrappers.WrapperValueFieldNumber, WireFormat.WireType.LengthDelimited)) } }; /// <summary> diff --git a/csharp/src/Google.Protobuf/Google.Protobuf.csproj b/csharp/src/Google.Protobuf/Google.Protobuf.csproj index 8c680d46..24fe7746 100644 --- a/csharp/src/Google.Protobuf/Google.Protobuf.csproj +++ b/csharp/src/Google.Protobuf/Google.Protobuf.csproj @@ -84,7 +84,11 @@ <Compile Include="FieldCodec.cs" />
<Compile Include="FrameworkPortability.cs" />
<Compile Include="IDeepCloneable.cs" />
+ <Compile Include="InvalidJsonException.cs" />
<Compile Include="JsonFormatter.cs" />
+ <Compile Include="JsonParser.cs" />
+ <Compile Include="JsonToken.cs" />
+ <Compile Include="JsonTokenizer.cs" />
<Compile Include="MessageExtensions.cs" />
<Compile Include="IMessage.cs" />
<Compile Include="InvalidProtocolBufferException.cs" />
@@ -130,7 +134,9 @@ <Compile Include="WellKnownTypes\Timestamp.cs" />
<Compile Include="WellKnownTypes\TimestampPartial.cs" />
<Compile Include="WellKnownTypes\Type.cs" />
+ <Compile Include="WellKnownTypes\ValuePartial.cs" />
<Compile Include="WellKnownTypes\Wrappers.cs" />
+ <Compile Include="WellKnownTypes\WrappersPartial.cs" />
<Compile Include="WireFormat.cs" />
</ItemGroup>
<ItemGroup>
diff --git a/csharp/src/Google.Protobuf/InvalidJsonException.cs b/csharp/src/Google.Protobuf/InvalidJsonException.cs new file mode 100644 index 00000000..b5434201 --- /dev/null +++ b/csharp/src/Google.Protobuf/InvalidJsonException.cs @@ -0,0 +1,53 @@ +#region Copyright notice and license +// Protocol Buffers - Google's data interchange format +// Copyright 2015 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#endregion + +using System.IO; + +namespace Google.Protobuf +{ + /// <summary> + /// Thrown when an attempt is made to parse invalid JSON, e.g. using + /// a non-string property key, or including a redundant comma. Parsing a protocol buffer + /// message represented in JSON using <see cref="JsonParser"/> can throw both this + /// exception and <see cref="InvalidProtocolBufferException"/> depending on the situation. This + /// exception is only thrown for "pure JSON" errors, whereas <c>InvalidProtocolBufferException</c> + /// is thrown when the JSON may be valid in and of itself, but cannot be parsed as a protocol buffer + /// message. + /// </summary> + public sealed class InvalidJsonException : IOException + { + internal InvalidJsonException(string message) + : base(message) + { + } + } +}
\ No newline at end of file diff --git a/csharp/src/Google.Protobuf/InvalidProtocolBufferException.cs b/csharp/src/Google.Protobuf/InvalidProtocolBufferException.cs index 01d55395..cacda648 100644 --- a/csharp/src/Google.Protobuf/InvalidProtocolBufferException.cs +++ b/csharp/src/Google.Protobuf/InvalidProtocolBufferException.cs @@ -95,6 +95,13 @@ namespace Google.Protobuf "Use CodedInputStream.SetRecursionLimit() to increase the depth limit.");
}
+ internal static InvalidProtocolBufferException JsonRecursionLimitExceeded()
+ {
+ return new InvalidProtocolBufferException(
+ "Protocol message had too many levels of nesting. May be malicious. " +
+ "Use JsonParser.Settings to increase the depth limit.");
+ }
+
internal static InvalidProtocolBufferException SizeLimitExceeded()
{
return new InvalidProtocolBufferException(
diff --git a/csharp/src/Google.Protobuf/JsonFormatter.cs b/csharp/src/Google.Protobuf/JsonFormatter.cs index 3f9bd478..2070f62b 100644 --- a/csharp/src/Google.Protobuf/JsonFormatter.cs +++ b/csharp/src/Google.Protobuf/JsonFormatter.cs @@ -189,6 +189,7 @@ namespace Google.Protobuf } // Converted from src/google/protobuf/util/internal/utility.cc ToCamelCase + // TODO: Use the new field in FieldDescriptor. internal static string ToCamelCase(string input) { bool capitalizeNext = false; @@ -382,10 +383,19 @@ namespace Google.Protobuf WriteNull(builder); return; } - // For wrapper types, the value will be the (possibly boxed) "native" value, - // so we can write it as if we were unconditionally writing the Value field for the wrapper type. + // For wrapper types, the value will either be the (possibly boxed) "native" value, + // or the message itself if we're formatting it at the top level (e.g. just calling ToString on the object itself). + // If it's the message form, we can extract the value first, which *will* be the (possibly boxed) native value, + // and then proceed, writing it as if we were definitely in a field. (We never need to wrap it in an extra string... + // WriteValue will do the right thing.) + // TODO: Detect this differently when we have dynamic messages. if (descriptor.File == Int32Value.Descriptor.File) { + if (value is IMessage) + { + var message = (IMessage) value; + value = message.Descriptor.Fields[Wrappers.WrapperValueFieldNumber].Accessor.GetValue(message); + } WriteValue(builder, value); return; } @@ -750,7 +760,6 @@ namespace Google.Protobuf private readonly bool formatDefaultValues; - /// <summary> /// Whether fields whose values are the default for the field type (e.g. 0 for integers) /// should be formatted (true) or omitted (false). diff --git a/csharp/src/Google.Protobuf/JsonParser.cs b/csharp/src/Google.Protobuf/JsonParser.cs new file mode 100644 index 00000000..c34f84fb --- /dev/null +++ b/csharp/src/Google.Protobuf/JsonParser.cs @@ -0,0 +1,823 @@ +#region Copyright notice and license +// Protocol Buffers - Google's data interchange format +// Copyright 2015 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#endregion + +using Google.Protobuf.Reflection; +using Google.Protobuf.WellKnownTypes; +using System; +using System.Collections; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; +using System.Text.RegularExpressions; + +namespace Google.Protobuf +{ + /// <summary> + /// Reflection-based converter from JSON to messages. + /// </summary> + /// <remarks> + /// <para> + /// Instances of this class are thread-safe, with no mutable state. + /// </para> + /// <para> + /// This is a simple start to get JSON parsing working. As it's reflection-based, + /// it's not as quick as baking calls into generated messages - but is a simpler implementation. + /// (This code is generally not heavily optimized.) + /// </para> + /// </remarks> + public sealed class JsonParser + { + // Note: using 0-9 instead of \d to ensure no non-ASCII digits. + // This regex isn't a complete validator, but will remove *most* invalid input. We rely on parsing to do the rest. + private static readonly Regex TimestampRegex = new Regex(@"^(?<datetime>[0-9]{4}-[01][0-9]-[0-3][0-9]T[012][0-9]:[0-5][0-9]:[0-5][0-9])(?<subseconds>\.[0-9]{1,9})?(?<offset>(Z|[+-][0-1][0-9]:[0-5][0-9]))$", FrameworkPortability.CompiledRegexWhereAvailable); + private static readonly Regex DurationRegex = new Regex(@"^(?<sign>-)?(?<int>[0-9]{1,12})(?<subseconds>\.[0-9]{1,9})?s$", FrameworkPortability.CompiledRegexWhereAvailable); + private static readonly int[] SubsecondScalingFactors = { 0, 100000000, 100000000, 10000000, 1000000, 100000, 10000, 1000, 100, 10, 1 }; + private static readonly char[] FieldMaskPathSeparators = new[] { ',' }; + + private static readonly JsonParser defaultInstance = new JsonParser(Settings.Default); + + // TODO: Consider introducing a class containing parse state of the parser, tokenizer and depth. That would simplify these handlers + // and the signatures of various methods. + private static readonly Dictionary<string, Action<JsonParser, IMessage, JsonTokenizer>> + WellKnownTypeHandlers = new Dictionary<string, Action<JsonParser, IMessage, JsonTokenizer>> + { + { Timestamp.Descriptor.FullName, (parser, message, tokenizer) => MergeTimestamp(message, tokenizer.Next()) }, + { Duration.Descriptor.FullName, (parser, message, tokenizer) => MergeDuration(message, tokenizer.Next()) }, + { Value.Descriptor.FullName, (parser, message, tokenizer) => parser.MergeStructValue(message, tokenizer) }, + { ListValue.Descriptor.FullName, (parser, message, tokenizer) => + parser.MergeRepeatedField(message, message.Descriptor.Fields[ListValue.ValuesFieldNumber], tokenizer) }, + { Struct.Descriptor.FullName, (parser, message, tokenizer) => parser.MergeStruct(message, tokenizer) }, + { FieldMask.Descriptor.FullName, (parser, message, tokenizer) => MergeFieldMask(message, tokenizer.Next()) }, + { Int32Value.Descriptor.FullName, MergeWrapperField }, + { Int64Value.Descriptor.FullName, MergeWrapperField }, + { UInt32Value.Descriptor.FullName, MergeWrapperField }, + { UInt64Value.Descriptor.FullName, MergeWrapperField }, + { FloatValue.Descriptor.FullName, MergeWrapperField }, + { DoubleValue.Descriptor.FullName, MergeWrapperField }, + { BytesValue.Descriptor.FullName, MergeWrapperField }, + { StringValue.Descriptor.FullName, MergeWrapperField } + }; + + // Convenience method to avoid having to repeat the same code multiple times in the above + // dictionary initialization. + private static void MergeWrapperField(JsonParser parser, IMessage message, JsonTokenizer tokenizer) + { + parser.MergeField(message, message.Descriptor.Fields[Wrappers.WrapperValueFieldNumber], tokenizer); + } + + /// <summary> + /// Returns a formatter using the default settings. + /// </summary> + public static JsonParser Default { get { return defaultInstance; } } + + private readonly Settings settings; + + /// <summary> + /// Creates a new formatted with the given settings. + /// </summary> + /// <param name="settings">The settings.</param> + public JsonParser(Settings settings) + { + this.settings = settings; + } + + /// <summary> + /// Parses <paramref name="json"/> and merges the information into the given message. + /// </summary> + /// <param name="message">The message to merge the JSON information into.</param> + /// <param name="json">The JSON to parse.</param> + internal void Merge(IMessage message, string json) + { + Merge(message, new StringReader(json)); + } + + /// <summary> + /// Parses JSON read from <paramref name="jsonReader"/> and merges the information into the given message. + /// </summary> + /// <param name="message">The message to merge the JSON information into.</param> + /// <param name="jsonReader">Reader providing the JSON to parse.</param> + internal void Merge(IMessage message, TextReader jsonReader) + { + var tokenizer = new JsonTokenizer(jsonReader); + Merge(message, tokenizer); + var lastToken = tokenizer.Next(); + if (lastToken != JsonToken.EndDocument) + { + throw new InvalidProtocolBufferException("Expected end of JSON after object"); + } + } + + /// <summary> + /// Merges the given message using data from the given tokenizer. In most cases, the next + /// token should be a "start object" token, but wrapper types and nullity can invalidate + /// that assumption. This is implemented as an LL(1) recursive descent parser over the stream + /// of tokens provided by the tokenizer. This token stream is assumed to be valid JSON, with the + /// tokenizer performing that validation - but not every token stream is valid "protobuf JSON". + /// </summary> + private void Merge(IMessage message, JsonTokenizer tokenizer) + { + if (tokenizer.ObjectDepth > settings.RecursionLimit) + { + throw InvalidProtocolBufferException.JsonRecursionLimitExceeded(); + } + if (message.Descriptor.IsWellKnownType) + { + Action<JsonParser, IMessage, JsonTokenizer> handler; + if (WellKnownTypeHandlers.TryGetValue(message.Descriptor.FullName, out handler)) + { + handler(this, message, tokenizer); + return; + } + // Well-known types with no special handling continue in the normal way. + } + var token = tokenizer.Next(); + if (token.Type != JsonToken.TokenType.StartObject) + { + throw new InvalidProtocolBufferException("Expected an object"); + } + var descriptor = message.Descriptor; + var jsonFieldMap = descriptor.Fields.ByJsonName(); + while (true) + { + token = tokenizer.Next(); + if (token.Type == JsonToken.TokenType.EndObject) + { + return; + } + if (token.Type != JsonToken.TokenType.Name) + { + throw new InvalidOperationException("Unexpected token type " + token.Type); + } + string name = token.StringValue; + FieldDescriptor field; + if (jsonFieldMap.TryGetValue(name, out field)) + { + MergeField(message, field, tokenizer); + } + else + { + // TODO: Is this what we want to do? If not, we'll need to skip the value, + // which may be an object or array. (We might want to put code in the tokenizer + // to do that.) + throw new InvalidProtocolBufferException("Unknown field: " + name); + } + } + } + + private void MergeField(IMessage message, FieldDescriptor field, JsonTokenizer tokenizer) + { + var token = tokenizer.Next(); + if (token.Type == JsonToken.TokenType.Null) + { + // Note: different from Java API, which just ignores it. + // TODO: Bring it more in line? Discuss... + field.Accessor.Clear(message); + return; + } + tokenizer.PushBack(token); + + if (field.IsMap) + { + MergeMapField(message, field, tokenizer); + } + else if (field.IsRepeated) + { + MergeRepeatedField(message, field, tokenizer); + } + else + { + var value = ParseSingleValue(field, tokenizer); + field.Accessor.SetValue(message, value); + } + } + + private void MergeRepeatedField(IMessage message, FieldDescriptor field, JsonTokenizer tokenizer) + { + var token = tokenizer.Next(); + if (token.Type != JsonToken.TokenType.StartArray) + { + throw new InvalidProtocolBufferException("Repeated field value was not an array. Token type: " + token.Type); + } + + IList list = (IList) field.Accessor.GetValue(message); + while (true) + { + token = tokenizer.Next(); + if (token.Type == JsonToken.TokenType.EndArray) + { + return; + } + tokenizer.PushBack(token); + list.Add(ParseSingleValue(field, tokenizer)); + } + } + + private void MergeMapField(IMessage message, FieldDescriptor field, JsonTokenizer tokenizer) + { + // Map fields are always objects, even if the values are well-known types: ParseSingleValue handles those. + var token = tokenizer.Next(); + if (token.Type != JsonToken.TokenType.StartObject) + { + throw new InvalidProtocolBufferException("Expected an object to populate a map"); + } + + var type = field.MessageType; + var keyField = type.FindFieldByNumber(1); + var valueField = type.FindFieldByNumber(2); + if (keyField == null || valueField == null) + { + throw new InvalidProtocolBufferException("Invalid map field: " + field.FullName); + } + IDictionary dictionary = (IDictionary) field.Accessor.GetValue(message); + + while (true) + { + token = tokenizer.Next(); + if (token.Type == JsonToken.TokenType.EndObject) + { + return; + } + object key = ParseMapKey(keyField, token.StringValue); + object value = ParseSingleValue(valueField, tokenizer); + // TODO: Null handling + dictionary[key] = value; + } + } + + private object ParseSingleValue(FieldDescriptor field, JsonTokenizer tokenizer) + { + var token = tokenizer.Next(); + if (token.Type == JsonToken.TokenType.Null) + { + if (field.FieldType == FieldType.Message && field.MessageType.FullName == Value.Descriptor.FullName) + { + return new Value { NullValue = NullValue.NULL_VALUE }; + } + return null; + } + + var fieldType = field.FieldType; + if (fieldType == FieldType.Message) + { + // Parse wrapper types as their constituent types. + // TODO: What does this mean for null? + // TODO: Detect this differently when we have dynamic messages, and put it in one place... + if (field.MessageType.IsWellKnownType && field.MessageType.File == Int32Value.Descriptor.File) + { + field = field.MessageType.Fields[Wrappers.WrapperValueFieldNumber]; + fieldType = field.FieldType; + } + else + { + // TODO: Merge the current value in message? (Public API currently doesn't make this relevant as we don't expose merging.) + tokenizer.PushBack(token); + IMessage subMessage = NewMessageForField(field); + Merge(subMessage, tokenizer); + return subMessage; + } + } + + switch (token.Type) + { + case JsonToken.TokenType.True: + case JsonToken.TokenType.False: + if (fieldType == FieldType.Bool) + { + return token.Type == JsonToken.TokenType.True; + } + // Fall through to "we don't support this type for this case"; could duplicate the behaviour of the default + // case instead, but this way we'd only need to change one place. + goto default; + case JsonToken.TokenType.StringValue: + return ParseSingleStringValue(field, token.StringValue); + // Note: not passing the number value itself here, as we may end up storing the string value in the token too. + case JsonToken.TokenType.Number: + return ParseSingleNumberValue(field, token); + case JsonToken.TokenType.Null: + throw new NotImplementedException("Haven't worked out what to do for null yet"); + default: + throw new InvalidProtocolBufferException("Unsupported JSON token type " + token.Type + " for field type " + fieldType); + } + } + + /// <summary> + /// Parses <paramref name="json"/> into a new message. + /// </summary> + /// <typeparam name="T">The type of message to create.</typeparam> + /// <param name="json">The JSON to parse.</param> + /// <exception cref="InvalidJsonException">The JSON does not comply with RFC 7159</exception> + /// <exception cref="InvalidProtocolBufferException">The JSON does not represent a Protocol Buffers message correctly</exception> + public T Parse<T>(string json) where T : IMessage, new() + { + return Parse<T>(new StringReader(json)); + } + + /// <summary> + /// Parses JSON read from <paramref name="jsonReader"/> into a new message. + /// </summary> + /// <typeparam name="T">The type of message to create.</typeparam> + /// <param name="jsonReader">Reader providing the JSON to parse.</param> + /// <exception cref="InvalidJsonException">The JSON does not comply with RFC 7159</exception> + /// <exception cref="InvalidProtocolBufferException">The JSON does not represent a Protocol Buffers message correctly</exception> + public T Parse<T>(TextReader jsonReader) where T : IMessage, new() + { + T message = new T(); + Merge(message, jsonReader); + return message; + } + + private void MergeStructValue(IMessage message, JsonTokenizer tokenizer) + { + var firstToken = tokenizer.Next(); + var fields = message.Descriptor.Fields; + switch (firstToken.Type) + { + case JsonToken.TokenType.Null: + fields[Value.NullValueFieldNumber].Accessor.SetValue(message, 0); + return; + case JsonToken.TokenType.StringValue: + fields[Value.StringValueFieldNumber].Accessor.SetValue(message, firstToken.StringValue); + return; + case JsonToken.TokenType.Number: + fields[Value.NumberValueFieldNumber].Accessor.SetValue(message, firstToken.NumberValue); + return; + case JsonToken.TokenType.False: + case JsonToken.TokenType.True: + fields[Value.BoolValueFieldNumber].Accessor.SetValue(message, firstToken.Type == JsonToken.TokenType.True); + return; + case JsonToken.TokenType.StartObject: + { + var field = fields[Value.StructValueFieldNumber]; + var structMessage = NewMessageForField(field); + tokenizer.PushBack(firstToken); + Merge(structMessage, tokenizer); + field.Accessor.SetValue(message, structMessage); + return; + } + case JsonToken.TokenType.StartArray: + { + var field = fields[Value.ListValueFieldNumber]; + var list = NewMessageForField(field); + tokenizer.PushBack(firstToken); + Merge(list, tokenizer); + field.Accessor.SetValue(message, list); + return; + } + default: + throw new InvalidOperationException("Unexpected token type: " + firstToken.Type); + } + } + + private void MergeStruct(IMessage message, JsonTokenizer tokenizer) + { + var token = tokenizer.Next(); + if (token.Type != JsonToken.TokenType.StartObject) + { + throw new InvalidProtocolBufferException("Expected object value for Struct"); + } + tokenizer.PushBack(token); + + var field = message.Descriptor.Fields[Struct.FieldsFieldNumber]; + MergeMapField(message, field, tokenizer); + } + + #region Utility methods which don't depend on the state (or settings) of the parser. + private static object ParseMapKey(FieldDescriptor field, string keyText) + { + switch (field.FieldType) + { + case FieldType.Bool: + if (keyText == "true") + { + return true; + } + if (keyText == "false") + { + return false; + } + throw new InvalidProtocolBufferException("Invalid string for bool map key: " + keyText); + case FieldType.String: + return keyText; + case FieldType.Int32: + case FieldType.SInt32: + case FieldType.SFixed32: + return ParseNumericString(keyText, int.Parse, false); + case FieldType.UInt32: + case FieldType.Fixed32: + return ParseNumericString(keyText, uint.Parse, false); + case FieldType.Int64: + case FieldType.SInt64: + case FieldType.SFixed64: + return ParseNumericString(keyText, long.Parse, false); + case FieldType.UInt64: + case FieldType.Fixed64: + return ParseNumericString(keyText, ulong.Parse, false); + default: + throw new InvalidProtocolBufferException("Invalid field type for map: " + field.FieldType); + } + } + + private static object ParseSingleNumberValue(FieldDescriptor field, JsonToken token) + { + double value = token.NumberValue; + checked + { + // TODO: Validate that it's actually an integer, possibly in terms of the textual representation? + try + { + switch (field.FieldType) + { + case FieldType.Int32: + case FieldType.SInt32: + case FieldType.SFixed32: + return (int) value; + case FieldType.UInt32: + case FieldType.Fixed32: + return (uint) value; + case FieldType.Int64: + case FieldType.SInt64: + case FieldType.SFixed64: + return (long) value; + case FieldType.UInt64: + case FieldType.Fixed64: + return (ulong) value; + case FieldType.Double: + return value; + case FieldType.Float: + if (double.IsNaN(value)) + { + return float.NaN; + } + if (value > float.MaxValue || value < float.MinValue) + { + if (double.IsPositiveInfinity(value)) + { + return float.PositiveInfinity; + } + if (double.IsNegativeInfinity(value)) + { + return float.NegativeInfinity; + } + throw new InvalidProtocolBufferException("Value out of range: " + value); + } + return (float) value; + default: + throw new InvalidProtocolBufferException("Unsupported conversion from JSON number for field type " + field.FieldType); + } + } + catch (OverflowException) + { + throw new InvalidProtocolBufferException("Value out of range: " + value); + } + } + } + + private static object ParseSingleStringValue(FieldDescriptor field, string text) + { + switch (field.FieldType) + { + case FieldType.String: + return text; + case FieldType.Bytes: + return ByteString.FromBase64(text); + case FieldType.Int32: + case FieldType.SInt32: + case FieldType.SFixed32: + return ParseNumericString(text, int.Parse, false); + case FieldType.UInt32: + case FieldType.Fixed32: + return ParseNumericString(text, uint.Parse, false); + case FieldType.Int64: + case FieldType.SInt64: + case FieldType.SFixed64: + return ParseNumericString(text, long.Parse, false); + case FieldType.UInt64: + case FieldType.Fixed64: + return ParseNumericString(text, ulong.Parse, false); + case FieldType.Double: + double d = ParseNumericString(text, double.Parse, true); + // double.Parse can return +/- infinity on Mono for non-infinite values which are out of range for double. + if (double.IsInfinity(d) && !text.Contains("Infinity")) + { + throw new InvalidProtocolBufferException("Invalid numeric value: " + text); + } + return d; + case FieldType.Float: + float f = ParseNumericString(text, float.Parse, true); + // float.Parse can return +/- infinity on Mono for non-infinite values which are out of range for float. + if (float.IsInfinity(f) && !text.Contains("Infinity")) + { + throw new InvalidProtocolBufferException("Invalid numeric value: " + text); + } + return f; + case FieldType.Enum: + var enumValue = field.EnumType.FindValueByName(text); + if (enumValue == null) + { + throw new InvalidProtocolBufferException("Invalid enum value: " + text + " for enum type: " + field.EnumType.FullName); + } + // Just return it as an int, and let the CLR convert it. + return enumValue.Number; + default: + throw new InvalidProtocolBufferException("Unsupported conversion from JSON string for field type " + field.FieldType); + } + } + + /// <summary> + /// Creates a new instance of the message type for the given field. + /// This method is mostly extracted so we can replace it in one go when we work out + /// what we want to do instead of Activator.CreateInstance. + /// </summary> + private static IMessage NewMessageForField(FieldDescriptor field) + { + // TODO: Create an instance in a better way ? + // (We could potentially add a Parser property to MessageDescriptor... see issue 806.) + return (IMessage) Activator.CreateInstance(field.MessageType.GeneratedType); + } + + private static T ParseNumericString<T>(string text, Func<string, NumberStyles, IFormatProvider, T> parser, bool floatingPoint) + { + // TODO: Prohibit leading zeroes (but allow 0!) + // TODO: Validate handling of "Infinity" etc. (Should be case sensitive, no leading whitespace etc) + // Can't prohibit this with NumberStyles. + if (text.StartsWith("+")) + { + throw new InvalidProtocolBufferException("Invalid numeric value: " + text); + } + if (text.StartsWith("0") && text.Length > 1) + { + if (text[1] >= '0' && text[1] <= '9') + { + throw new InvalidProtocolBufferException("Invalid numeric value: " + text); + } + } + else if (text.StartsWith("-0") && text.Length > 2) + { + if (text[2] >= '0' && text[2] <= '9') + { + throw new InvalidProtocolBufferException("Invalid numeric value: " + text); + } + } + try + { + var styles = floatingPoint + ? NumberStyles.AllowLeadingSign | NumberStyles.AllowDecimalPoint | NumberStyles.AllowExponent + : NumberStyles.AllowLeadingSign; + return parser(text, styles, CultureInfo.InvariantCulture); + } + catch (FormatException) + { + throw new InvalidProtocolBufferException("Invalid numeric value for type: " + text); + } + catch (OverflowException) + { + throw new InvalidProtocolBufferException("Value out of range: " + text); + } + } + + private static void MergeTimestamp(IMessage message, JsonToken token) + { + if (token.Type != JsonToken.TokenType.StringValue) + { + throw new InvalidProtocolBufferException("Expected string value for Timestamp"); + } + var match = TimestampRegex.Match(token.StringValue); + if (!match.Success) + { + throw new InvalidProtocolBufferException("Invalid Timestamp value: " + token.StringValue); + } + var dateTime = match.Groups["datetime"].Value; + var subseconds = match.Groups["subseconds"].Value; + var offset = match.Groups["offset"].Value; + + try + { + DateTime parsed = DateTime.ParseExact( + dateTime, + "yyyy-MM-dd'T'HH:mm:ss", + CultureInfo.InvariantCulture, + DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal); + // TODO: It would be nice not to have to create all these objects... easy to optimize later though. + Timestamp timestamp = Timestamp.FromDateTime(parsed); + int nanosToAdd = 0; + if (subseconds != "") + { + // This should always work, as we've got 1-9 digits. + int parsedFraction = int.Parse(subseconds.Substring(1), CultureInfo.InvariantCulture); + nanosToAdd = parsedFraction * SubsecondScalingFactors[subseconds.Length]; + } + int secondsToAdd = 0; + if (offset != "Z") + { + // This is the amount we need to *subtract* from the local time to get to UTC - hence - => +1 and vice versa. + int sign = offset[0] == '-' ? 1 : -1; + int hours = int.Parse(offset.Substring(1, 2), CultureInfo.InvariantCulture); + int minutes = int.Parse(offset.Substring(4, 2)); + int totalMinutes = hours * 60 + minutes; + if (totalMinutes > 18 * 60) + { + throw new InvalidProtocolBufferException("Invalid Timestamp value: " + token.StringValue); + } + if (totalMinutes == 0 && sign == 1) + { + // This is an offset of -00:00, which means "unknown local offset". It makes no sense for a timestamp. + throw new InvalidProtocolBufferException("Invalid Timestamp value: " + token.StringValue); + } + // We need to *subtract* the offset from local time to get UTC. + secondsToAdd = sign * totalMinutes * 60; + } + // Ensure we've got the right signs. Currently unnecessary, but easy to do. + if (secondsToAdd < 0 && nanosToAdd > 0) + { + secondsToAdd++; + nanosToAdd = nanosToAdd - Duration.NanosecondsPerSecond; + } + if (secondsToAdd != 0 || nanosToAdd != 0) + { + timestamp += new Duration { Nanos = nanosToAdd, Seconds = secondsToAdd }; + // The resulting timestamp after offset change would be out of our expected range. Currently the Timestamp message doesn't validate this + // anywhere, but we shouldn't parse it. + if (timestamp.Seconds < Timestamp.UnixSecondsAtBclMinValue || timestamp.Seconds > Timestamp.UnixSecondsAtBclMaxValue) + { + throw new InvalidProtocolBufferException("Invalid Timestamp value: " + token.StringValue); + } + } + message.Descriptor.Fields[Timestamp.SecondsFieldNumber].Accessor.SetValue(message, timestamp.Seconds); + message.Descriptor.Fields[Timestamp.NanosFieldNumber].Accessor.SetValue(message, timestamp.Nanos); + } + catch (FormatException) + { + throw new InvalidProtocolBufferException("Invalid Timestamp value: " + token.StringValue); + } + } + + private static void MergeDuration(IMessage message, JsonToken token) + { + if (token.Type != JsonToken.TokenType.StringValue) + { + throw new InvalidProtocolBufferException("Expected string value for Duration"); + } + var match = DurationRegex.Match(token.StringValue); + if (!match.Success) + { + throw new InvalidProtocolBufferException("Invalid Duration value: " + token.StringValue); + } + var sign = match.Groups["sign"].Value; + var secondsText = match.Groups["int"].Value; + // Prohibit leading insignficant zeroes + if (secondsText[0] == '0' && secondsText.Length > 1) + { + throw new InvalidProtocolBufferException("Invalid Duration value: " + token.StringValue); + } + var subseconds = match.Groups["subseconds"].Value; + var multiplier = sign == "-" ? -1 : 1; + + try + { + long seconds = long.Parse(secondsText, CultureInfo.InvariantCulture); + int nanos = 0; + if (subseconds != "") + { + // This should always work, as we've got 1-9 digits. + int parsedFraction = int.Parse(subseconds.Substring(1)); + nanos = parsedFraction * SubsecondScalingFactors[subseconds.Length]; + } + if (seconds >= Duration.MaxSeconds) + { + // Allow precisely 315576000000 seconds, but prohibit even 1ns more. + if (seconds > Duration.MaxSeconds || nanos > 0) + { + throw new InvalidProtocolBufferException("Invalid Duration value: " + token.StringValue); + } + } + message.Descriptor.Fields[Duration.SecondsFieldNumber].Accessor.SetValue(message, seconds * multiplier); + message.Descriptor.Fields[Duration.NanosFieldNumber].Accessor.SetValue(message, nanos * multiplier); + } + catch (FormatException) + { + throw new InvalidProtocolBufferException("Invalid Duration value: " + token.StringValue); + } + } + + private static void MergeFieldMask(IMessage message, JsonToken token) + { + if (token.Type != JsonToken.TokenType.StringValue) + { + throw new InvalidProtocolBufferException("Expected string value for FieldMask"); + } + // TODO: Do we *want* to remove empty entries? Probably okay to treat "" as "no paths", but "foo,,bar"? + string[] jsonPaths = token.StringValue.Split(FieldMaskPathSeparators, StringSplitOptions.RemoveEmptyEntries); + IList messagePaths = (IList) message.Descriptor.Fields[FieldMask.PathsFieldNumber].Accessor.GetValue(message); + foreach (var path in jsonPaths) + { + messagePaths.Add(ToSnakeCase(path)); + } + } + + // Ported from src/google/protobuf/util/internal/utility.cc + private static string ToSnakeCase(string text) + { + var builder = new StringBuilder(text.Length * 2); + bool wasNotUnderscore = false; // Initialize to false for case 1 (below) + bool wasNotCap = false; + + for (int i = 0; i < text.Length; i++) + { + char c = text[i]; + if (c >= 'A' && c <= 'Z') // ascii_isupper + { + // Consider when the current character B is capitalized: + // 1) At beginning of input: "B..." => "b..." + // (e.g. "Biscuit" => "biscuit") + // 2) Following a lowercase: "...aB..." => "...a_b..." + // (e.g. "gBike" => "g_bike") + // 3) At the end of input: "...AB" => "...ab" + // (e.g. "GoogleLAB" => "google_lab") + // 4) Followed by a lowercase: "...ABc..." => "...a_bc..." + // (e.g. "GBike" => "g_bike") + if (wasNotUnderscore && // case 1 out + (wasNotCap || // case 2 in, case 3 out + (i + 1 < text.Length && // case 3 out + (text[i + 1] >= 'a' && text[i + 1] <= 'z')))) // ascii_islower(text[i + 1]) + { // case 4 in + // We add an underscore for case 2 and case 4. + builder.Append('_'); + } + // ascii_tolower, but we already know that c *is* an upper case ASCII character... + builder.Append((char) (c + 'a' - 'A')); + wasNotUnderscore = true; + wasNotCap = false; + } + else + { + builder.Append(c); + wasNotUnderscore = c != '_'; + wasNotCap = true; + } + } + return builder.ToString(); + } + #endregion + + /// <summary> + /// Settings controlling JSON parsing. + /// </summary> + public sealed class Settings + { + private static readonly Settings defaultInstance = new Settings(CodedInputStream.DefaultRecursionLimit); + + private readonly int recursionLimit; + + /// <summary> + /// Default settings, as used by <see cref="JsonParser.Default"/> + /// </summary> + public static Settings Default { get { return defaultInstance; } } + + /// <summary> + /// The maximum depth of messages to parse. Note that this limit only applies to parsing + /// messages, not collections - so a message within a collection within a message only counts as + /// depth 2, not 3. + /// </summary> + public int RecursionLimit { get { return recursionLimit; } } + + /// <summary> + /// Creates a new <see cref="Settings"/> object with the specified recursion limit. + /// </summary> + /// <param name="recursionLimit">The maximum depth of messages to parse</param> + public Settings(int recursionLimit) + { + this.recursionLimit = recursionLimit; + } + } + } +} diff --git a/csharp/src/Google.Protobuf/JsonToken.cs b/csharp/src/Google.Protobuf/JsonToken.cs new file mode 100644 index 00000000..6c0138cc --- /dev/null +++ b/csharp/src/Google.Protobuf/JsonToken.cs @@ -0,0 +1,166 @@ +#region Copyright notice and license +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#endregion + +using System; + +namespace Google.Protobuf +{ + internal sealed class JsonToken : IEquatable<JsonToken> + { + // Tokens with no value can be reused. + private static readonly JsonToken _true = new JsonToken(TokenType.True); + private static readonly JsonToken _false = new JsonToken(TokenType.False); + private static readonly JsonToken _null = new JsonToken(TokenType.Null); + private static readonly JsonToken startObject = new JsonToken(TokenType.StartObject); + private static readonly JsonToken endObject = new JsonToken(TokenType.EndObject); + private static readonly JsonToken startArray = new JsonToken(TokenType.StartArray); + private static readonly JsonToken endArray = new JsonToken(TokenType.EndArray); + private static readonly JsonToken endDocument = new JsonToken(TokenType.EndDocument); + + internal static JsonToken Null { get { return _null; } } + internal static JsonToken False { get { return _false; } } + internal static JsonToken True { get { return _true; } } + internal static JsonToken StartObject{ get { return startObject; } } + internal static JsonToken EndObject { get { return endObject; } } + internal static JsonToken StartArray { get { return startArray; } } + internal static JsonToken EndArray { get { return endArray; } } + internal static JsonToken EndDocument { get { return endDocument; } } + + internal static JsonToken Name(string name) + { + return new JsonToken(TokenType.Name, stringValue: name); + } + + internal static JsonToken Value(string value) + { + return new JsonToken(TokenType.StringValue, stringValue: value); + } + + internal static JsonToken Value(double value) + { + return new JsonToken(TokenType.Number, numberValue: value); + } + + internal enum TokenType + { + Null, + False, + True, + StringValue, + Number, + Name, + StartObject, + EndObject, + StartArray, + EndArray, + EndDocument + } + + // A value is a string, number, array, object, null, true or false + // Arrays and objects have start/end + // A document consists of a value + // Objects are name/value sequences. + + private readonly TokenType type; + private readonly string stringValue; + private readonly double numberValue; + + internal TokenType Type { get { return type; } } + internal string StringValue { get { return stringValue; } } + internal double NumberValue { get { return numberValue; } } + + private JsonToken(TokenType type, string stringValue = null, double numberValue = 0) + { + this.type = type; + this.stringValue = stringValue; + this.numberValue = numberValue; + } + + public override bool Equals(object obj) + { + return Equals(obj as JsonToken); + } + + public override int GetHashCode() + { + unchecked + { + int hash = 17; + hash = hash * 31 + (int) type; + hash = hash * 31 + stringValue == null ? 0 : stringValue.GetHashCode(); + hash = hash * 31 + numberValue.GetHashCode(); + return hash; + } + } + + public override string ToString() + { + switch (type) + { + case TokenType.Null: + return "null"; + case TokenType.True: + return "true"; + case TokenType.False: + return "false"; + case TokenType.Name: + return "name (" + stringValue + ")"; + case TokenType.StringValue: + return "value (" + stringValue + ")"; + case TokenType.Number: + return "number (" + numberValue + ")"; + case TokenType.StartObject: + return "start-object"; + case TokenType.EndObject: + return "end-object"; + case TokenType.StartArray: + return "start-array"; + case TokenType.EndArray: + return "end-array"; + case TokenType.EndDocument: + return "end-document"; + default: + throw new InvalidOperationException("Token is of unknown type " + type); + } + } + + public bool Equals(JsonToken other) + { + if (ReferenceEquals(other, null)) + { + return false; + } + // Note use of other.numberValue.Equals rather than ==, so that NaN compares appropriately. + return other.type == type && other.stringValue == stringValue && other.numberValue.Equals(numberValue); + } + } +} diff --git a/csharp/src/Google.Protobuf/JsonTokenizer.cs b/csharp/src/Google.Protobuf/JsonTokenizer.cs new file mode 100644 index 00000000..6589427a --- /dev/null +++ b/csharp/src/Google.Protobuf/JsonTokenizer.cs @@ -0,0 +1,670 @@ +#region Copyright notice and license +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#endregion +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; + +namespace Google.Protobuf +{ + /// <summary> + /// Simple but strict JSON tokenizer, rigidly following RFC 7159. + /// </summary> + /// <remarks> + /// <para> + /// This tokenizer is stateful, and only returns "useful" tokens - names, values etc. + /// It does not create tokens for the separator between names and values, or for the comma + /// between values. It validates the token stream as it goes - so callers can assume that the + /// tokens it produces are appropriate. For example, it would never produce "start object, end array." + /// </para> + /// <para>Not thread-safe.</para> + /// </remarks> + internal sealed class JsonTokenizer + { + // The set of states in which a value is valid next token. + private static readonly State ValueStates = State.ArrayStart | State.ArrayAfterComma | State.ObjectAfterColon | State.StartOfDocument; + + private readonly Stack<ContainerType> containerStack = new Stack<ContainerType>(); + private readonly PushBackReader reader; + private JsonToken bufferedToken; + private State state; + private int objectDepth = 0; + + /// <summary> + /// Returns the depth of the stack, purely in objects (not collections). + /// Informally, this is the number of remaining unclosed '{' characters we have. + /// </summary> + internal int ObjectDepth { get { return objectDepth; } } + + internal JsonTokenizer(TextReader reader) + { + this.reader = new PushBackReader(reader); + state = State.StartOfDocument; + containerStack.Push(ContainerType.Document); + } + + // TODO: Why do we allow a different token to be pushed back? It might be better to always remember the previous + // token returned, and allow a parameterless Rewind() method (which could only be called once, just like the current PushBack). + internal void PushBack(JsonToken token) + { + if (bufferedToken != null) + { + throw new InvalidOperationException("Can't push back twice"); + } + bufferedToken = token; + if (token.Type == JsonToken.TokenType.StartObject) + { + objectDepth--; + } + else if (token.Type == JsonToken.TokenType.EndObject) + { + objectDepth++; + } + } + + /// <summary> + /// Returns the next JSON token in the stream. An EndDocument token is returned to indicate the end of the stream, + /// after which point <c>Next()</c> should not be called again. + /// </summary> + /// <remarks> + /// This method essentially just loops through characters skipping whitespace, validating and + /// changing state (e.g. from ObjectBeforeColon to ObjectAfterColon) + /// until it reaches something which will be a genuine token (e.g. a start object, or a value) at which point + /// it returns the token. Although the method is large, it would be relatively hard to break down further... most + /// of it is the large switch statement, which sometimes returns and sometimes doesn't. + /// </remarks> + /// <returns>The next token in the stream. This is never null.</returns> + /// <exception cref="InvalidOperationException">This method is called after an EndDocument token has been returned</exception> + /// <exception cref="InvalidJsonException">The input text does not comply with RFC 7159</exception> + internal JsonToken Next() + { + if (bufferedToken != null) + { + var ret = bufferedToken; + bufferedToken = null; + if (ret.Type == JsonToken.TokenType.StartObject) + { + objectDepth++; + } + else if (ret.Type == JsonToken.TokenType.EndObject) + { + objectDepth--; + } + return ret; + } + if (state == State.ReaderExhausted) + { + throw new InvalidOperationException("Next() called after end of document"); + } + while (true) + { + var next = reader.Read(); + if (next == null) + { + ValidateState(State.ExpectedEndOfDocument, "Unexpected end of document in state: "); + state = State.ReaderExhausted; + return JsonToken.EndDocument; + } + switch (next.Value) + { + // Skip whitespace between tokens + case ' ': + case '\t': + case '\r': + case '\n': + break; + case ':': + ValidateState(State.ObjectBeforeColon, "Invalid state to read a colon: "); + state = State.ObjectAfterColon; + break; + case ',': + ValidateState(State.ObjectAfterProperty | State.ArrayAfterValue, "Invalid state to read a colon: "); + state = state == State.ObjectAfterProperty ? State.ObjectAfterComma : State.ArrayAfterComma; + break; + case '"': + string stringValue = ReadString(); + if ((state & (State.ObjectStart | State.ObjectAfterComma)) != 0) + { + state = State.ObjectBeforeColon; + return JsonToken.Name(stringValue); + } + else + { + ValidateAndModifyStateForValue("Invalid state to read a double quote: "); + return JsonToken.Value(stringValue); + } + case '{': + ValidateState(ValueStates, "Invalid state to read an open brace: "); + state = State.ObjectStart; + containerStack.Push(ContainerType.Object); + objectDepth++; + return JsonToken.StartObject; + case '}': + ValidateState(State.ObjectAfterProperty | State.ObjectStart, "Invalid state to read a close brace: "); + PopContainer(); + objectDepth--; + return JsonToken.EndObject; + case '[': + ValidateState(ValueStates, "Invalid state to read an open square bracket: "); + state = State.ArrayStart; + containerStack.Push(ContainerType.Array); + return JsonToken.StartArray; + case ']': + ValidateState(State.ArrayAfterValue | State.ArrayStart, "Invalid state to read a close square bracket: "); + PopContainer(); + return JsonToken.EndArray; + case 'n': // Start of null + ConsumeLiteral("null"); + ValidateAndModifyStateForValue("Invalid state to read a null literal: "); + return JsonToken.Null; + case 't': // Start of true + ConsumeLiteral("true"); + ValidateAndModifyStateForValue("Invalid state to read a true literal: "); + return JsonToken.True; + case 'f': // Start of false + ConsumeLiteral("false"); + ValidateAndModifyStateForValue("Invalid state to read a false literal: "); + return JsonToken.False; + case '-': // Start of a number + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + double number = ReadNumber(next.Value); + ValidateAndModifyStateForValue("Invalid state to read a number token: "); + return JsonToken.Value(number); + default: + throw new InvalidJsonException("Invalid first character of token: " + next.Value); + } + } + } + + private void ValidateState(State validStates, string errorPrefix) + { + if ((validStates & state) == 0) + { + throw reader.CreateException(errorPrefix + state); + } + } + + /// <summary> + /// Reads a string token. It is assumed that the opening " has already been read. + /// </summary> + private string ReadString() + { + var value = new StringBuilder(); + bool haveHighSurrogate = false; + while (true) + { + char c = reader.ReadOrFail("Unexpected end of text while reading string"); + if (c < ' ') + { + throw reader.CreateException(string.Format(CultureInfo.InvariantCulture, "Invalid character in string literal: U+{0:x4}", (int) c)); + } + if (c == '"') + { + if (haveHighSurrogate) + { + throw reader.CreateException("Invalid use of surrogate pair code units"); + } + return value.ToString(); + } + if (c == '\\') + { + c = ReadEscapedCharacter(); + } + // TODO: Consider only allowing surrogate pairs that are either both escaped, + // or both not escaped. It would be a very odd text stream that contained a "lone" high surrogate + // followed by an escaped low surrogate or vice versa... and that couldn't even be represented in UTF-8. + if (haveHighSurrogate != char.IsLowSurrogate(c)) + { + throw reader.CreateException("Invalid use of surrogate pair code units"); + } + haveHighSurrogate = char.IsHighSurrogate(c); + value.Append(c); + } + } + + /// <summary> + /// Reads an escaped character. It is assumed that the leading backslash has already been read. + /// </summary> + private char ReadEscapedCharacter() + { + char c = reader.ReadOrFail("Unexpected end of text while reading character escape sequence"); + switch (c) + { + case 'n': + return '\n'; + case '\\': + return '\\'; + case 'b': + return '\b'; + case 'f': + return '\f'; + case 'r': + return '\r'; + case 't': + return '\t'; + case '"': + return '"'; + case '/': + return '/'; + case 'u': + return ReadUnicodeEscape(); + default: + throw reader.CreateException(string.Format(CultureInfo.InvariantCulture, "Invalid character in character escape sequence: U+{0:x4}", (int) c)); + } + } + + /// <summary> + /// Reads an escaped Unicode 4-nybble hex sequence. It is assumed that the leading \u has already been read. + /// </summary> + private char ReadUnicodeEscape() + { + int result = 0; + for (int i = 0; i < 4; i++) + { + char c = reader.ReadOrFail("Unexpected end of text while reading Unicode escape sequence"); + int nybble; + if (c >= '0' && c <= '9') + { + nybble = c - '0'; + } + else if (c >= 'a' && c <= 'f') + { + nybble = c - 'a' + 10; + } + else if (c >= 'A' && c <= 'F') + { + nybble = c - 'A' + 10; + } + else + { + throw reader.CreateException(string.Format(CultureInfo.InvariantCulture, "Invalid character in character escape sequence: U+{0:x4}", (int) c)); + } + result = (result << 4) + nybble; + } + return (char) result; + } + + /// <summary> + /// Consumes a text-only literal, throwing an exception if the read text doesn't match it. + /// It is assumed that the first letter of the literal has already been read. + /// </summary> + private void ConsumeLiteral(string text) + { + for (int i = 1; i < text.Length; i++) + { + char? next = reader.Read(); + if (next == null) + { + throw reader.CreateException("Unexpected end of text while reading literal token " + text); + } + if (next.Value != text[i]) + { + throw reader.CreateException("Unexpected character while reading literal token " + text); + } + } + } + + private double ReadNumber(char initialCharacter) + { + StringBuilder builder = new StringBuilder(); + if (initialCharacter == '-') + { + builder.Append("-"); + } + else + { + reader.PushBack(initialCharacter); + } + // Each method returns the character it read that doesn't belong in that part, + // so we know what to do next, including pushing the character back at the end. + // null is returned for "end of text". + char? next = ReadInt(builder); + if (next == '.') + { + next = ReadFrac(builder); + } + if (next == 'e' || next == 'E') + { + next = ReadExp(builder); + } + // If we read a character which wasn't part of the number, push it back so we can read it again + // to parse the next token. + if (next != null) + { + reader.PushBack(next.Value); + } + + // TODO: What exception should we throw if the value can't be represented as a double? + try + { + return double.Parse(builder.ToString(), + NumberStyles.AllowLeadingSign | NumberStyles.AllowDecimalPoint | NumberStyles.AllowExponent, + CultureInfo.InvariantCulture); + } + catch (OverflowException) + { + throw reader.CreateException("Numeric value out of range: " + builder); + } + } + + private char? ReadInt(StringBuilder builder) + { + char first = reader.ReadOrFail("Invalid numeric literal"); + if (first < '0' || first > '9') + { + throw reader.CreateException("Invalid numeric literal"); + } + builder.Append(first); + int digitCount; + char? next = ConsumeDigits(builder, out digitCount); + if (first == '0' && digitCount != 0) + { + throw reader.CreateException("Invalid numeric literal: leading 0 for non-zero value."); + } + return next; + } + + private char? ReadFrac(StringBuilder builder) + { + builder.Append('.'); // Already consumed this + int digitCount; + char? next = ConsumeDigits(builder, out digitCount); + if (digitCount == 0) + { + throw reader.CreateException("Invalid numeric literal: fraction with no trailing digits"); + } + return next; + } + + private char? ReadExp(StringBuilder builder) + { + builder.Append('E'); // Already consumed this (or 'e') + char? next = reader.Read(); + if (next == null) + { + throw reader.CreateException("Invalid numeric literal: exponent with no trailing digits"); + } + if (next == '-' || next == '+') + { + builder.Append(next.Value); + } + else + { + reader.PushBack(next.Value); + } + int digitCount; + next = ConsumeDigits(builder, out digitCount); + if (digitCount == 0) + { + throw reader.CreateException("Invalid numeric literal: exponent without value"); + } + return next; + } + + private char? ConsumeDigits(StringBuilder builder, out int count) + { + count = 0; + while (true) + { + char? next = reader.Read(); + if (next == null || next.Value < '0' || next.Value > '9') + { + return next; + } + count++; + builder.Append(next.Value); + } + } + + /// <summary> + /// Validates that we're in a valid state to read a value (using the given error prefix if necessary) + /// and changes the state to the appropriate one, e.g. ObjectAfterColon to ObjectAfterProperty. + /// </summary> + private void ValidateAndModifyStateForValue(string errorPrefix) + { + ValidateState(ValueStates, errorPrefix); + switch (state) + { + case State.StartOfDocument: + state = State.ExpectedEndOfDocument; + return; + case State.ObjectAfterColon: + state = State.ObjectAfterProperty; + return; + case State.ArrayStart: + case State.ArrayAfterComma: + state = State.ArrayAfterValue; + return; + default: + throw new InvalidOperationException("ValidateAndModifyStateForValue does not handle all value states (and should)"); + } + } + + /// <summary> + /// Pops the top-most container, and sets the state to the appropriate one for the end of a value + /// in the parent container. + /// </summary> + private void PopContainer() + { + containerStack.Pop(); + var parent = containerStack.Peek(); + switch (parent) + { + case ContainerType.Object: + state = State.ObjectAfterProperty; + break; + case ContainerType.Array: + state = State.ArrayAfterValue; + break; + case ContainerType.Document: + state = State.ExpectedEndOfDocument; + break; + default: + throw new InvalidOperationException("Unexpected container type: " + parent); + } + } + + private enum ContainerType + { + Document, Object, Array + } + + /// <summary> + /// Possible states of the tokenizer. + /// </summary> + /// <remarks> + /// <para>This is a flags enum purely so we can simply and efficiently represent a set of valid states + /// for checking.</para> + /// <para> + /// Each is documented with an example, + /// where ^ represents the current position within the text stream. The examples all use string values, + /// but could be any value, including nested objects/arrays. + /// The complete state of the tokenizer also includes a stack to indicate the contexts (arrays/objects). + /// Any additional notional state of "AfterValue" indicates that a value has been completed, at which + /// point there's an immediate transition to ExpectedEndOfDocument, ObjectAfterProperty or ArrayAfterValue. + /// </para> + /// <para> + /// These states were derived manually by reading RFC 7159 carefully. + /// </para> + /// </remarks> + [Flags] + private enum State + { + /// <summary> + /// ^ { "foo": "bar" } + /// Before the value in a document. Next states: ObjectStart, ArrayStart, "AfterValue" + /// </summary> + StartOfDocument = 1 << 0, + /// <summary> + /// { "foo": "bar" } ^ + /// After the value in a document. Next states: ReaderExhausted + /// </summary> + ExpectedEndOfDocument = 1 << 1, + /// <summary> + /// { "foo": "bar" } ^ (and already read to the end of the reader) + /// Terminal state. + /// </summary> + ReaderExhausted = 1 << 2, + /// <summary> + /// { ^ "foo": "bar" } + /// Before the *first* property in an object. + /// Next states: + /// "AfterValue" (empty object) + /// ObjectBeforeColon (read a name) + /// </summary> + ObjectStart = 1 << 3, + /// <summary> + /// { "foo" ^ : "bar", "x": "y" } + /// Next state: ObjectAfterColon + /// </summary> + ObjectBeforeColon = 1 << 4, + /// <summary> + /// { "foo" : ^ "bar", "x": "y" } + /// Before any property other than the first in an object. + /// (Equivalently: after any property in an object) + /// Next states: + /// "AfterValue" (value is simple) + /// ObjectStart (value is object) + /// ArrayStart (value is array) + /// </summary> + ObjectAfterColon = 1 << 5, + /// <summary> + /// { "foo" : "bar" ^ , "x" : "y" } + /// At the end of a property, so expecting either a comma or end-of-object + /// Next states: ObjectAfterComma or "AfterValue" + /// </summary> + ObjectAfterProperty = 1 << 6, + /// <summary> + /// { "foo":"bar", ^ "x":"y" } + /// Read the comma after the previous property, so expecting another property. + /// This is like ObjectStart, but closing brace isn't valid here + /// Next state: ObjectBeforeColon. + /// </summary> + ObjectAfterComma = 1 << 7, + /// <summary> + /// [ ^ "foo", "bar" ] + /// Before the *first* value in an array. + /// Next states: + /// "AfterValue" (read a value) + /// "AfterValue" (end of array; will pop stack) + /// </summary> + ArrayStart = 1 << 8, + /// <summary> + /// [ "foo" ^ , "bar" ] + /// After any value in an array, so expecting either a comma or end-of-array + /// Next states: ArrayAfterComma or "AfterValue" + /// </summary> + ArrayAfterValue = 1 << 9, + /// <summary> + /// [ "foo", ^ "bar" ] + /// After a comma in an array, so there *must* be another value (simple or complex). + /// Next states: "AfterValue" (simple value), StartObject, StartArray + /// </summary> + ArrayAfterComma = 1 << 10 + } + + /// <summary> + /// Wrapper around a text reader allowing small amounts of buffering and location handling. + /// </summary> + private class PushBackReader + { + // TODO: Add locations for errors etc. + + private readonly TextReader reader; + + internal PushBackReader(TextReader reader) + { + // TODO: Wrap the reader in a BufferedReader? + this.reader = reader; + } + + /// <summary> + /// The buffered next character, if we have one. + /// </summary> + private char? nextChar; + + /// <summary> + /// Returns the next character in the stream, or null if we have reached the end. + /// </summary> + /// <returns></returns> + internal char? Read() + { + if (nextChar != null) + { + char? tmp = nextChar; + nextChar = null; + return tmp; + } + int next = reader.Read(); + return next == -1 ? null : (char?) next; + } + + internal char ReadOrFail(string messageOnFailure) + { + char? next = Read(); + if (next == null) + { + throw CreateException(messageOnFailure); + } + return next.Value; + } + + internal void PushBack(char c) + { + if (nextChar != null) + { + throw new InvalidOperationException("Cannot push back when already buffering a character"); + } + nextChar = c; + } + + /// <summary> + /// Creates a new exception appropriate for the current state of the reader. + /// </summary> + internal InvalidJsonException CreateException(string message) + { + // TODO: Keep track of and use the location. + return new InvalidJsonException(message); + } + } + } +} diff --git a/csharp/src/Google.Protobuf/MessageParser.cs b/csharp/src/Google.Protobuf/MessageParser.cs index 6a6f1017..8f2717c5 100644 --- a/csharp/src/Google.Protobuf/MessageParser.cs +++ b/csharp/src/Google.Protobuf/MessageParser.cs @@ -142,5 +142,19 @@ namespace Google.Protobuf message.MergeFrom(input); return message; } + + /// <summary> + /// Parses a message from the given JSON. + /// </summary> + /// <param name="json">The JSON to parse.</param> + /// <returns>The parsed message.</returns> + /// <exception cref="InvalidJsonException">The JSON does not comply with RFC 7159</exception> + /// <exception cref="InvalidProtocolBufferException">The JSON does not represent a Protocol Buffers message correctly</exception> + public T ParseJson(string json) + { + T message = factory(); + JsonParser.Default.Merge(message, json); + return message; + } } } diff --git a/csharp/src/Google.Protobuf/Properties/AssemblyInfo.cs b/csharp/src/Google.Protobuf/Properties/AssemblyInfo.cs index 9a484ad6..225ac0dd 100644 --- a/csharp/src/Google.Protobuf/Properties/AssemblyInfo.cs +++ b/csharp/src/Google.Protobuf/Properties/AssemblyInfo.cs @@ -46,7 +46,10 @@ using System.Security; [assembly: AssemblyCopyright("Copyright © 2015")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
+
+#if !NCRUNCH
[assembly: AllowPartiallyTrustedCallers]
+#endif
#if SIGNED
[assembly: InternalsVisibleTo("Google.Protobuf.Test, PublicKey=" +
diff --git a/csharp/src/Google.Protobuf/Reflection/DescriptorProtoFile.cs b/csharp/src/Google.Protobuf/Reflection/DescriptorProtoFile.cs index ee43bec2..25a03538 100644 --- a/csharp/src/Google.Protobuf/Reflection/DescriptorProtoFile.cs +++ b/csharp/src/Google.Protobuf/Reflection/DescriptorProtoFile.cs @@ -2695,6 +2695,7 @@ namespace Google.Protobuf.Reflection { /// main code generators in each language (without additional plugins). /// Generic services were the only kind of service generation supported by /// early versions of google.protobuf. + /// /// Generic services are now considered deprecated in favor of using plugins /// that generate code specific to your particular RPC system. Therefore, /// these default to false. Old code which depends on generic services should @@ -3173,6 +3174,7 @@ namespace Google.Protobuf.Reflection { /// This is provided for backwards-compatibility with the MessageSet wire /// format. You should not use this for any other reason: It's less /// efficient, has fewer features, and is more complicated. + /// /// The message must be defined exactly as follows: /// message Foo { /// option message_set_wire_format = true; @@ -3180,8 +3182,10 @@ namespace Google.Protobuf.Reflection { /// } /// Note that the message cannot have any defined fields; MessageSets only /// have extensions. + /// /// All extensions of your type must be singular messages; e.g. they cannot /// be int32s, enums, or repeated messages. + /// /// Because this is an option, the above two restrictions are not enforced by /// the protocol compiler. /// </summary> @@ -3229,6 +3233,7 @@ namespace Google.Protobuf.Reflection { /// <summary> /// Whether the message is an automatically generated map entry type for the /// maps field. + /// /// For maps fields: /// map<KeyType, ValueType> map_field = 1; /// The parsed descriptor looks like: @@ -3238,10 +3243,12 @@ namespace Google.Protobuf.Reflection { /// optional ValueType value = 2; /// } /// repeated MapFieldEntry map_field = 1; + /// /// Implementations may choose not to generate the map_entry=true message, but /// use a native map in the target language to hold the keys and values. /// The reflection APIs in such implementions still need to work as /// if the field is a repeated message field. + /// /// NOTE: Do not set the option in .proto files. Always use the maps syntax /// instead. The option should only be implicitly set by the proto compiler /// parser. @@ -3483,16 +3490,19 @@ namespace Google.Protobuf.Reflection { /// fields. It means that when the outer message is initially parsed, the /// inner message's contents will not be parsed but instead stored in encoded /// form. The inner message will actually be parsed when it is first accessed. + /// /// This is only a hint. Implementations are free to choose whether to use /// eager or lazy parsing regardless of the value of this option. However, /// setting this option true suggests that the protocol author believes that /// using lazy parsing on this field is worth the additional bookkeeping /// overhead typically needed to implement it. + /// /// This option does not affect the public interface of any generated code; /// all method signatures remain the same. Furthermore, thread-safety of the /// interface is not affected by this option; const methods remain safe to /// call from multiple threads concurrently, while non-const methods continue /// to require exclusive access. + /// /// Note that implementations may choose not to check required fields within /// a lazy sub-message. That is, calling IsInitialized() on the outher message /// may return true even if the inner message has missing required fields. @@ -4747,6 +4757,7 @@ namespace Google.Protobuf.Reflection { /// corresponds to a particular definition. This information is intended /// to be useful to IDEs, code indexers, documentation generators, and similar /// tools. + /// /// For example, say we have a file like: /// message Foo { /// optional string foo = 1; @@ -4762,6 +4773,7 @@ namespace Google.Protobuf.Reflection { /// [c,d) [ 4, 0, 2, 0, 5 ] The type (string). /// [e,f) [ 4, 0, 2, 0, 1 ] The name (foo). /// [g,h) [ 4, 0, 2, 0, 3 ] The number (1). + /// /// Notes: /// - A location may refer to a repeated field itself (i.e. not to any /// particular index within it). This is used whenever a set of elements are @@ -4889,6 +4901,7 @@ namespace Google.Protobuf.Reflection { /// <summary> /// Identifies which part of the FileDescriptorProto was defined at this /// location. + /// /// Each element is a field number or an index. They form a path from /// the root FileDescriptorProto to the place where the definition. For /// example, this path: @@ -4903,6 +4916,7 @@ namespace Google.Protobuf.Reflection { /// repeated FieldDescriptorProto field = 2; /// and FieldDescriptorProto.name has field number 1: /// optional string name = 1; + /// /// Thus, the above path gives the location of a field name. If we removed /// the last element: /// [ 4, 3, 2, 7 ] @@ -4936,31 +4950,41 @@ namespace Google.Protobuf.Reflection { /// If this SourceCodeInfo represents a complete declaration, these are any /// comments appearing before and after the declaration which appear to be /// attached to the declaration. + /// /// A series of line comments appearing on consecutive lines, with no other /// tokens appearing on those lines, will be treated as a single comment. + /// /// leading_detached_comments will keep paragraphs of comments that appear /// before (but not connected to) the current element. Each paragraph, /// separated by empty lines, will be one comment element in the repeated /// field. + /// /// Only the comment content is provided; comment markers (e.g. //) are /// stripped out. For block comments, leading whitespace and an asterisk /// will be stripped from the beginning of each line other than the first. /// Newlines are included in the output. + /// /// Examples: + /// /// optional int32 foo = 1; // Comment attached to foo. /// // Comment attached to bar. /// optional int32 bar = 2; + /// /// optional string baz = 3; /// // Comment attached to baz. /// // Another line attached to baz. + /// /// // Comment attached to qux. /// // /// // Another line attached to qux. /// optional double qux = 4; + /// /// // Detached comment for corge. This is not leading or trailing comments /// // to qux or corge because there are blank lines separating it from /// // both. + /// /// // Detached comment for corge paragraph 2. + /// /// optional string corge = 5; /// /* Block comment attached /// * to corge. Leading asterisks @@ -4968,6 +4992,7 @@ namespace Google.Protobuf.Reflection { /// /* Block comment attached to /// * grault. */ /// optional int32 grault = 6; + /// /// // ignored detached comments. /// </summary> public string LeadingComments { diff --git a/csharp/src/Google.Protobuf/Reflection/MessageDescriptor.cs b/csharp/src/Google.Protobuf/Reflection/MessageDescriptor.cs index 82901f1b..e599998e 100644 --- a/csharp/src/Google.Protobuf/Reflection/MessageDescriptor.cs +++ b/csharp/src/Google.Protobuf/Reflection/MessageDescriptor.cs @@ -62,6 +62,7 @@ namespace Google.Protobuf.Reflection private readonly IList<EnumDescriptor> enumTypes; private readonly IList<FieldDescriptor> fieldsInDeclarationOrder; private readonly IList<FieldDescriptor> fieldsInNumberOrder; + private readonly IDictionary<string, FieldDescriptor> jsonFieldMap; private readonly FieldCollection fields; private readonly IList<OneofDescriptor> oneofs; // CLR representation of the type described by this descriptor, if any. @@ -95,6 +96,8 @@ namespace Google.Protobuf.Reflection (field, index) => new FieldDescriptor(field, file, this, index, generatedCodeInfo == null ? null : generatedCodeInfo.PropertyNames[index])); fieldsInNumberOrder = new ReadOnlyCollection<FieldDescriptor>(fieldsInDeclarationOrder.OrderBy(field => field.FieldNumber).ToArray()); + // TODO: Use field => field.Proto.JsonName when we're confident it's appropriate. (And then use it in the formatter, too.) + jsonFieldMap = new ReadOnlyDictionary<string, FieldDescriptor>(fieldsInNumberOrder.ToDictionary(field => JsonFormatter.ToCamelCase(field.Name))); file.DescriptorPool.AddSymbol(this); fields = new FieldCollection(this); } @@ -255,6 +258,18 @@ namespace Google.Protobuf.Reflection return messageDescriptor.fieldsInNumberOrder; } + // TODO: consider making this public in the future. (Being conservative for now...) + + /// <value> + /// Returns a read-only dictionary mapping the field names in this message as they're used + /// in the JSON representation to the field descriptors. For example, a field <c>foo_bar</c> + /// in the message would result in an entry with a key <c>fooBar</c>. + /// </value> + internal IDictionary<string, FieldDescriptor> ByJsonName() + { + return messageDescriptor.jsonFieldMap; + } + /// <summary> /// Retrieves the descriptor for the field with the given number. /// </summary> diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/Any.cs b/csharp/src/Google.Protobuf/WellKnownTypes/Any.cs index 4a704018..64cbd33d 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/Any.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/Any.cs @@ -43,25 +43,30 @@ namespace Google.Protobuf.WellKnownTypes { /// <summary> /// `Any` contains an arbitrary serialized message along with a URL /// that describes the type of the serialized message. + /// /// JSON /// ==== /// The JSON representation of an `Any` value uses the regular /// representation of the deserialized, embedded message, with an /// additional field `@type` which contains the type URL. Example: + /// /// package google.profile; /// message Person { /// string first_name = 1; /// string last_name = 2; /// } + /// /// { /// "@type": "type.googleapis.com/google.profile.Person", /// "firstName": <string>, /// "lastName": <string> /// } + /// /// If the embedded message type is well-known and has a custom JSON /// representation, that representation will be embedded adding a field /// `value` which holds the custom JSON in addition to the the `@type` /// field. Example (for message [google.protobuf.Duration][google.protobuf.Duration]): + /// /// { /// "@type": "type.googleapis.com/google.protobuf.Duration", /// "value": "1.212s" @@ -101,8 +106,10 @@ namespace Google.Protobuf.WellKnownTypes { /// <summary> /// A URL/resource name whose content describes the type of the /// serialized message. + /// /// For URLs which use the schema `http`, `https`, or no schema, the /// following restrictions and interpretations apply: + /// /// * If no schema is provided, `https` is assumed. /// * The last segment of the URL's path must represent the fully /// qualified name of the type (as in `path/google.protobuf.Duration`). @@ -113,6 +120,7 @@ namespace Google.Protobuf.WellKnownTypes { /// lookup. Therefore, binary compatibility needs to be preserved /// on changes to types. (Use versioned type names to manage /// breaking changes.) + /// /// Schemas other than `http`, `https` (or the empty schema) might be /// used with implementation specific semantics. /// </summary> diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/Api.cs b/csharp/src/Google.Protobuf/WellKnownTypes/Api.cs index 2c64314d..899613f9 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/Api.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/Api.cs @@ -139,12 +139,14 @@ namespace Google.Protobuf.WellKnownTypes { /// outlined below. If the field is not empty, the version in the /// package name will be verified to be consistent with what is /// provided here. + /// /// The versioning schema uses [semantic /// versioning](http://semver.org) where the major version number /// indicates a breaking change and the minor version an additive, /// non-breaking change. Both version numbers are signals to users /// what to expect from different versions, and should be carefully /// chosen based on the product plan. + /// /// The major version is also reflected in the package name of the /// API, which must end in `v<major-version>`, as in /// `google.feature.v1`. For major versions 0 and 1, the suffix can @@ -629,16 +631,21 @@ namespace Google.Protobuf.WellKnownTypes { /// Declares an API to be included in this API. The including API must /// redeclare all the methods from the included API, but documentation /// and options are inherited as follows: + /// /// - If after comment and whitespace stripping, the documentation /// string of the redeclared method is empty, it will be inherited /// from the original method. + /// /// - Each annotation belonging to the service config (http, /// visibility) which is not set in the redeclared method will be /// inherited. + /// /// - If an http annotation is inherited, the path pattern will be /// modified as follows. Any version prefix will be replaced by the /// version of the including API plus the [root][] path if specified. + /// /// Example of a simple mixin: + /// /// package google.acl.v1; /// service AccessControl { /// // Get the underlying ACL object. @@ -646,25 +653,31 @@ namespace Google.Protobuf.WellKnownTypes { /// option (google.api.http).get = "/v1/{resource=**}:getAcl"; /// } /// } + /// /// package google.storage.v2; /// service Storage { /// // (-- see AccessControl.GetAcl --) /// rpc GetAcl(GetAclRequest) returns (Acl); + /// /// // Get a data record. /// rpc GetData(GetDataRequest) returns (Data) { /// option (google.api.http).get = "/v2/{resource=**}"; /// } /// } + /// /// Example of a mixin configuration: + /// /// apis: /// - name: google.storage.v2.Storage /// mixins: /// - name: google.acl.v1.AccessControl + /// /// The mixin construct implies that all methods in `AccessControl` are /// also declared with same name and request/response types in /// `Storage`. A documentation generator or annotation processor will /// see the effective `Storage.GetAcl` method after inherting /// documentation and annotations as follows: + /// /// service Storage { /// // Get the underlying ACL object. /// rpc GetAcl(GetAclRequest) returns (Acl) { @@ -672,15 +685,20 @@ namespace Google.Protobuf.WellKnownTypes { /// } /// ... /// } + /// /// Note how the version in the path pattern changed from `v1` to `v2`. + /// /// If the `root` field in the mixin is specified, it should be a /// relative path under which inherited HTTP paths are placed. Example: + /// /// apis: /// - name: google.storage.v2.Storage /// mixins: /// - name: google.acl.v1.AccessControl /// root: acls + /// /// This implies the following inherited HTTP annotation: + /// /// service Storage { /// // Get the underlying ACL object. /// rpc GetAcl(GetAclRequest) returns (Acl) { diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/Duration.cs b/csharp/src/Google.Protobuf/WellKnownTypes/Duration.cs index 39251e2e..cc2dfd26 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/Duration.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/Duration.cs @@ -48,12 +48,16 @@ namespace Google.Protobuf.WellKnownTypes { /// or "month". It is related to Timestamp in that the difference between /// two Timestamp values is a Duration and it can be added or subtracted /// from a Timestamp. Range is approximately +-10,000 years. + /// /// Example 1: Compute Duration from two Timestamps in pseudo code. + /// /// Timestamp start = ...; /// Timestamp end = ...; /// Duration duration = ...; + /// /// duration.seconds = end.seconds - start.seconds; /// duration.nanos = end.nanos - start.nanos; + /// /// if (duration.seconds < 0 && duration.nanos > 0) { /// duration.seconds += 1; /// duration.nanos -= 1000000000; @@ -61,12 +65,16 @@ namespace Google.Protobuf.WellKnownTypes { /// duration.seconds -= 1; /// duration.nanos += 1000000000; /// } + /// /// Example 2: Compute Timestamp from Timestamp + Duration in pseudo code. + /// /// Timestamp start = ...; /// Duration duration = ...; /// Timestamp end = ...; + /// /// end.seconds = start.seconds + duration.seconds; /// end.nanos = start.nanos + duration.nanos; + /// /// if (end.nanos < 0) { /// end.seconds -= 1; /// end.nanos += 1000000000; diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/DurationPartial.cs b/csharp/src/Google.Protobuf/WellKnownTypes/DurationPartial.cs index 18ebefd2..324f48fc 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/DurationPartial.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/DurationPartial.cs @@ -48,6 +48,16 @@ namespace Google.Protobuf.WellKnownTypes public const int NanosecondsPerTick = 100; /// <summary> + /// The maximum permitted number of seconds. + /// </summary> + public const long MaxSeconds = 315576000000L; + + /// <summary> + /// The minimum permitted number of seconds. + /// </summary> + public const long MinSeconds = -315576000000L; + + /// <summary> /// Converts this <see cref="Duration"/> to a <see cref="TimeSpan"/>. /// </summary> /// <remarks>If the duration is not a precise number of ticks, it is truncated towards 0.</remarks> diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/Empty.cs b/csharp/src/Google.Protobuf/WellKnownTypes/Empty.cs index 18223a9e..f2d2de01 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/Empty.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/Empty.cs @@ -44,9 +44,11 @@ namespace Google.Protobuf.WellKnownTypes { /// A generic empty message that you can re-use to avoid defining duplicated /// empty messages in your APIs. A typical example is to use it as the request /// or the response type of an API method. For instance: + /// /// service Foo { /// rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); /// } + /// /// The JSON representation for `Empty` is empty JSON object `{}`. /// </summary> [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/FieldMask.cs b/csharp/src/Google.Protobuf/WellKnownTypes/FieldMask.cs index 33c814ef..e71e4af6 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/FieldMask.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/FieldMask.cs @@ -42,19 +42,25 @@ namespace Google.Protobuf.WellKnownTypes { #region Messages /// <summary> /// `FieldMask` represents a set of symbolic field paths, for example: + /// /// paths: "f.a" /// paths: "f.b.d" + /// /// Here `f` represents a field in some root message, `a` and `b` /// fields in the message found in `f`, and `d` a field found in the /// message in `f.b`. + /// /// Field masks are used to specify a subset of fields that should be /// returned by a get operation or modified by an update operation. /// Field masks also have a custom JSON encoding (see below). + /// /// # Field Masks in Projections + /// /// When used in the context of a projection, a response message or /// sub-message is filtered by the API to only contain those fields as /// specified in the mask. For example, if the mask in the previous /// example is applied to a response message as follows: + /// /// f { /// a : 22 /// b { @@ -64,20 +70,25 @@ namespace Google.Protobuf.WellKnownTypes { /// y : 13 /// } /// z: 8 + /// /// The result will not contain specific values for fields x,y and z /// (there value will be set to the default, and omitted in proto text /// output): + /// /// f { /// a : 22 /// b { /// d : 1 /// } /// } + /// /// A repeated field is not allowed except at the last position of a /// field mask. + /// /// If a FieldMask object is not present in a get operation, the /// operation applies to all fields (as if a FieldMask of all fields /// had been specified). + /// /// Note that a field mask does not necessarily applies to the /// top-level response message. In case of a REST get operation, the /// field mask applies directly to the response, but in case of a REST @@ -87,18 +98,22 @@ namespace Google.Protobuf.WellKnownTypes { /// clearly documented together with its declaration in the API. In /// any case, the effect on the returned resource/resources is required /// behavior for APIs. + /// /// # Field Masks in Update Operations + /// /// A field mask in update operations specifies which fields of the /// targeted resource are going to be updated. The API is required /// to only change the values of the fields as specified in the mask /// and leave the others untouched. If a resource is passed in to /// describe the updated values, the API ignores the values of all /// fields not covered by the mask. + /// /// In order to reset a field's value to the default, the field must /// be in the mask and set to the default value in the provided resource. /// Hence, in order to reset all fields of a resource, provide a default /// instance of the resource and set all fields in the mask, or do /// not provide a mask as described below. + /// /// If a field mask is not present on update, the operation applies to /// all fields (as if a field mask of all fields has been specified). /// Note that in the presence of schema evolution, this may mean that @@ -106,19 +121,26 @@ namespace Google.Protobuf.WellKnownTypes { /// the request will be reset to their default. If this is unwanted /// behavior, a specific service may require a client to always specify /// a field mask, producing an error if not. + /// /// As with get operations, the location of the resource which /// describes the updated values in the request message depends on the /// operation kind. In any case, the effect of the field mask is /// required to be honored by the API. + /// /// ## Considerations for HTTP REST + /// /// The HTTP kind of an update operation which uses a field mask must /// be set to PATCH instead of PUT in order to satisfy HTTP semantics /// (PUT must only be used for full updates). + /// /// # JSON Encoding of Field Masks + /// /// In JSON, a field mask is encoded as a single string where paths are /// separated by a comma. Fields name in each path are converted /// to/from lower-camel naming conventions. + /// /// As an example, consider the following message declarations: + /// /// message Profile { /// User user = 1; /// Photo photo = 2; @@ -127,12 +149,16 @@ namespace Google.Protobuf.WellKnownTypes { /// string display_name = 1; /// string address = 2; /// } + /// /// In proto a field mask for `Profile` may look as such: + /// /// mask { /// paths: "user.display_name" /// paths: "photo" /// } + /// /// In JSON, the same mask is represented as below: + /// /// { /// mask: "user.displayName,photo" /// } diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/Struct.cs b/csharp/src/Google.Protobuf/WellKnownTypes/Struct.cs index b2d7783c..8f8e917d 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/Struct.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/Struct.cs @@ -55,6 +55,7 @@ namespace Google.Protobuf.WellKnownTypes { /// <summary> /// `NullValue` is a singleton enumeration to represent the null value for the /// `Value` type union. + /// /// The JSON representation for `NullValue` is JSON `null`. /// </summary> public enum NullValue { @@ -74,6 +75,7 @@ namespace Google.Protobuf.WellKnownTypes { /// scripting languages like JS a struct is represented as an /// object. The details of that representation are described together /// with the proto support for the language. + /// /// The JSON representation for `Struct` is JSON object. /// </summary> [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] @@ -179,6 +181,7 @@ namespace Google.Protobuf.WellKnownTypes { /// null, a number, a string, a boolean, a recursive struct value, or a /// list of values. A producer of value is expected to set one of that /// variants, absence of any variant indicates an error. + /// /// The JSON representation for `Value` is JSON value. /// </summary> [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] @@ -491,6 +494,7 @@ namespace Google.Protobuf.WellKnownTypes { /// <summary> /// `ListValue` is a wrapper around a repeated field of values. + /// /// The JSON representation for `ListValue` is JSON array. /// </summary> [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/Timestamp.cs b/csharp/src/Google.Protobuf/WellKnownTypes/Timestamp.cs index f372f8fd..1044a683 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/Timestamp.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/Timestamp.cs @@ -53,30 +53,43 @@ namespace Google.Protobuf.WellKnownTypes { /// By restricting to that range, we ensure that we can convert to /// and from RFC 3339 date strings. /// See [https://www.ietf.org/rfc/rfc3339.txt](https://www.ietf.org/rfc/rfc3339.txt). + /// /// Example 1: Compute Timestamp from POSIX `time()`. + /// /// Timestamp timestamp; /// timestamp.set_seconds(time(NULL)); /// timestamp.set_nanos(0); + /// /// Example 2: Compute Timestamp from POSIX `gettimeofday()`. + /// /// struct timeval tv; /// gettimeofday(&tv, NULL); + /// /// Timestamp timestamp; /// timestamp.set_seconds(tv.tv_sec); /// timestamp.set_nanos(tv.tv_usec * 1000); + /// /// Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. + /// /// FILETIME ft; /// GetSystemTimeAsFileTime(&ft); /// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; + /// /// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z /// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. /// Timestamp timestamp; /// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); /// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); + /// /// Example 4: Compute Timestamp from Java `System.currentTimeMillis()`. + /// /// long millis = System.currentTimeMillis(); + /// /// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) /// .setNanos((int) ((millis % 1000) * 1000000)).build(); + /// /// Example 5: Compute Timestamp from current time in Python. + /// /// now = time.time() /// seconds = int(now) /// nanos = int((now - seconds) * 10**9) diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/TimestampPartial.cs b/csharp/src/Google.Protobuf/WellKnownTypes/TimestampPartial.cs index 68584358..d284acd6 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/TimestampPartial.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/TimestampPartial.cs @@ -38,6 +38,8 @@ namespace Google.Protobuf.WellKnownTypes { private static readonly DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); private static readonly long BclSecondsAtUnixEpoch = UnixEpoch.Ticks / TimeSpan.TicksPerSecond; + internal static readonly long UnixSecondsAtBclMinValue = -BclSecondsAtUnixEpoch; + internal static readonly long UnixSecondsAtBclMaxValue = (DateTime.MaxValue.Ticks / TimeSpan.TicksPerSecond) - BclSecondsAtUnixEpoch; /// <summary> /// Returns the difference between one <see cref="Timestamp"/> and another, as a <see cref="Duration"/>. diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/ValuePartial.cs b/csharp/src/Google.Protobuf/WellKnownTypes/ValuePartial.cs new file mode 100644 index 00000000..b4b6c05e --- /dev/null +++ b/csharp/src/Google.Protobuf/WellKnownTypes/ValuePartial.cs @@ -0,0 +1,99 @@ +#region Copyright notice and license +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#endregion + +namespace Google.Protobuf.WellKnownTypes +{ + public partial class Value + { + /// <summary> + /// Convenience method to create a Value message with a string value. + /// </summary> + /// <param name="value">Value to set for the StringValue property.</param> + /// <returns>A newly-created Value message with the given value.</returns> + public static Value ForString(string value) + { + Preconditions.CheckNotNull(value, "value"); + return new Value { StringValue = value }; + } + + /// <summary> + /// Convenience method to create a Value message with a number value. + /// </summary> + /// <param name="value">Value to set for the NumberValue property.</param> + /// <returns>A newly-created Value message with the given value.</returns> + public static Value ForNumber(double value) + { + return new Value { NumberValue = value }; + } + + /// <summary> + /// Convenience method to create a Value message with a Boolean value. + /// </summary> + /// <param name="value">Value to set for the BoolValue property.</param> + /// <returns>A newly-created Value message with the given value.</returns> + public static Value ForBool(bool value) + { + return new Value { BoolValue = value }; + } + + /// <summary> + /// Convenience method to create a Value message with a null initial value. + /// </summary> + /// <returns>A newly-created Value message a null initial value.</returns> + public static Value ForNull() + { + return new Value { NullValue = 0 }; + } + + /// <summary> + /// Convenience method to create a Value message with an initial list of values. + /// </summary> + /// <remarks>The values provided are not cloned; the references are copied directly.</remarks> + /// <returns>A newly-created Value message an initial list value.</returns> + public static Value ForList(params Value[] values) + { + Preconditions.CheckNotNull(values, "values"); + return new Value { ListValue = new ListValue { Values = { values } } }; + } + + /// <summary> + /// Convenience method to create a Value message with an initial struct value + /// </summary> + /// <remarks>The value provided is not cloned; the reference is copied directly.</remarks> + /// <returns>A newly-created Value message an initial struct value.</returns> + public static Value ForStruct(Struct value) + { + Preconditions.CheckNotNull(value, "value"); + return new Value { StructValue = value }; + } + } +} diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/Wrappers.cs b/csharp/src/Google.Protobuf/WellKnownTypes/Wrappers.cs index d26395f8..7fdcb11e 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/Wrappers.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/Wrappers.cs @@ -52,6 +52,7 @@ namespace Google.Protobuf.WellKnownTypes { #region Messages /// <summary> /// Wrapper message for `double`. + /// /// The JSON representation for `DoubleValue` is JSON number. /// </summary> [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] @@ -162,6 +163,7 @@ namespace Google.Protobuf.WellKnownTypes { /// <summary> /// Wrapper message for `float`. + /// /// The JSON representation for `FloatValue` is JSON number. /// </summary> [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] @@ -272,6 +274,7 @@ namespace Google.Protobuf.WellKnownTypes { /// <summary> /// Wrapper message for `int64`. + /// /// The JSON representation for `Int64Value` is JSON string. /// </summary> [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] @@ -382,6 +385,7 @@ namespace Google.Protobuf.WellKnownTypes { /// <summary> /// Wrapper message for `uint64`. + /// /// The JSON representation for `UInt64Value` is JSON string. /// </summary> [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] @@ -492,6 +496,7 @@ namespace Google.Protobuf.WellKnownTypes { /// <summary> /// Wrapper message for `int32`. + /// /// The JSON representation for `Int32Value` is JSON number. /// </summary> [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] @@ -602,6 +607,7 @@ namespace Google.Protobuf.WellKnownTypes { /// <summary> /// Wrapper message for `uint32`. + /// /// The JSON representation for `UInt32Value` is JSON number. /// </summary> [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] @@ -712,6 +718,7 @@ namespace Google.Protobuf.WellKnownTypes { /// <summary> /// Wrapper message for `bool`. + /// /// The JSON representation for `BoolValue` is JSON `true` and `false`. /// </summary> [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] @@ -822,6 +829,7 @@ namespace Google.Protobuf.WellKnownTypes { /// <summary> /// Wrapper message for `string`. + /// /// The JSON representation for `StringValue` is JSON string. /// </summary> [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] @@ -932,6 +940,7 @@ namespace Google.Protobuf.WellKnownTypes { /// <summary> /// Wrapper message for `bytes`. + /// /// The JSON representation for `BytesValue` is JSON string. /// </summary> [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/WrappersPartial.cs b/csharp/src/Google.Protobuf/WellKnownTypes/WrappersPartial.cs new file mode 100644 index 00000000..efc13a01 --- /dev/null +++ b/csharp/src/Google.Protobuf/WellKnownTypes/WrappersPartial.cs @@ -0,0 +1,42 @@ +#region Copyright notice and license +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#endregion + +namespace Google.Protobuf.WellKnownTypes +{ + public static partial class Wrappers + { + /// <summary> + /// Field number for the single "value" field in all wrapper types. + /// </summary> + internal const int WrapperValueFieldNumber = Int32Value.ValueFieldNumber; + } +} diff --git a/java/pom.xml b/java/pom.xml index 8b81b645..7b99c035 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -92,6 +92,7 @@ <arg value="../src/google/protobuf/timestamp.proto" /> <arg value="../src/google/protobuf/type.proto" /> <arg value="../src/google/protobuf/wrappers.proto" /> + <arg value="../src/google/protobuf/compiler/plugin.proto" /> </exec> </tasks> <sourceRoot>target/generated-sources</sourceRoot> @@ -186,6 +187,7 @@ <include>google/protobuf/timestamp.proto</include> <include>google/protobuf/type.proto</include> <include>google/protobuf/wrappers.proto</include> + <include>google/protobuf/compiler/plugin.proto</include> </includes> </resource> </resources> diff --git a/objectivec/DevTools/full_mac_build.sh b/objectivec/DevTools/full_mac_build.sh index c38fce7c..71d3fa45 100755 --- a/objectivec/DevTools/full_mac_build.sh +++ b/objectivec/DevTools/full_mac_build.sh @@ -26,10 +26,12 @@ OPTIONS: Issue a clean before the normal build. -a, --autogen Start by rerunning autogen & configure. - -r, --regenerate-descriptors + -r, --regenerate-cpp-descriptors The descriptor.proto is checked in generated, cause it to regenerate. -j #, --jobs # Force the number of parallel jobs (useful for debugging build issues). + --core-only + Skip some of the core protobuf build/checks to shorten the build time. --skip-xcode Skip the invoke of Xcode to test the runtime on both iOS and OS X. --skip-xcode-ios @@ -68,6 +70,7 @@ fi DO_AUTOGEN=no DO_CLEAN=no REGEN_CPP_DESCRIPTORS=no +CORE_ONLY=no DO_XCODE_IOS_TESTS=yes DO_XCODE_OSX_TESTS=yes while [[ $# != 0 ]]; do @@ -89,6 +92,9 @@ while [[ $# != 0 ]]; do shift NUM_MAKE_JOBS="${1}" ;; + --core-only ) + CORE_ONLY=yes + ;; --skip-xcode ) DO_XCODE_IOS_TESTS=no DO_XCODE_OSX_TESTS=no @@ -114,7 +120,7 @@ while [[ $# != 0 ]]; do done # Into the proto dir. -pushd "${ProtoRootDir}" +cd "${ProtoRootDir}" # if no Makefile, force the autogen. if [[ ! -f Makefile ]] ; then @@ -155,11 +161,20 @@ if [[ "${REGEN_CPP_DESCRIPTORS}" == "yes" ]] ; then ./generate_descriptor_proto.sh -j "${NUM_MAKE_JOBS}" fi -header "Building" -# Can't issue these together, when fully parallel, something sometimes chokes -# at random. -wrapped_make -j "${NUM_MAKE_JOBS}" all -wrapped_make -j "${NUM_MAKE_JOBS}" check +if [[ "${CORE_ONLY}" == "yes" ]] ; then + header "Building core Only" + wrapped_make -j "${NUM_MAKE_JOBS}" +else + header "Building" + # Can't issue these together, when fully parallel, something sometimes chokes + # at random. + wrapped_make -j "${NUM_MAKE_JOBS}" all + wrapped_make -j "${NUM_MAKE_JOBS}" check + # Fire off the conformance tests also. + cd conformance + wrapped_make -j "${NUM_MAKE_JOBS}" + cd .. +fi header "Ensuring the ObjC descriptors are current." # Find the newest input file (protos, compiler, and the generator script). @@ -199,19 +214,42 @@ if [[ "${DO_XCODE_IOS_TESTS}" == "yes" ]] ; then xcodebuild -project objectivec/ProtocolBuffers_iOS.xcodeproj -scheme ProtocolBuffers - # Don't need to worry about form factors or retina/non retina; - # just pick a mix of OS Versions and 32/64 bit. - -destination "platform=iOS Simulator,name=iPhone 4s,OS=7.1" # 32bit - -destination "platform=iOS Simulator,name=iPhone 6,OS=8.4" # 64bit - -destination "platform=iOS Simulator,name=iPad 2,OS=7.1" # 32bit - -destination "platform=iOS Simulator,name=iPad Air,OS=8.4" # 64bit ) + # Don't need to worry about form factors or retina/non retina; + # just pick a mix of OS Versions and 32/64 bit. + # NOTE: Different Xcode have different simulated hardware/os support. + readonly XCODE_VERSION_LINE="$(xcodebuild -version | grep Xcode\ )" + readonly XCODE_VERSION="${XCODE_VERSION_LINE/Xcode /}" # drop the prefix. + IOS_SIMULATOR_NAME="Simulator" + case "${XCODE_VERSION}" in + 6.* ) + XCODEBUILD_TEST_BASE_IOS+=( + -destination "platform=iOS Simulator,name=iPhone 4s,OS=7.1" # 32bit + -destination "platform=iOS Simulator,name=iPhone 6,OS=8.4" # 64bit + -destination "platform=iOS Simulator,name=iPad 2,OS=7.1" # 32bit + -destination "platform=iOS Simulator,name=iPad Air,OS=8.4" # 64bit + ) + IOS_SIMULATOR_NAME="iOS Simulator" + ;; + 7.* ) + XCODEBUILD_TEST_BASE_IOS+=( + -destination "platform=iOS Simulator,name=iPhone 4s,OS=8.1" # 32bit + -destination "platform=iOS Simulator,name=iPhone 6,OS=9.0" # 64bit + -destination "platform=iOS Simulator,name=iPad 2,OS=8.1" # 32bit + -destination "platform=iOS Simulator,name=iPad Air,OS=9.0" # 64bit + ) + ;; + * ) + echo "Time to update the simulator targets for Xcode ${XCODE_VERSION}" + exit 2 + ;; + esac header "Doing Xcode iOS build/tests - Debug" "${XCODEBUILD_TEST_BASE_IOS[@]}" -configuration Debug test header "Doing Xcode iOS build/tests - Release" "${XCODEBUILD_TEST_BASE_IOS[@]}" -configuration Release test # Don't leave the simulator in the developer's face. - killall "iOS Simulator" + killall "${IOS_SIMULATOR_NAME}" fi if [[ "${DO_XCODE_OSX_TESTS}" == "yes" ]] ; then XCODEBUILD_TEST_BASE_OSX=( diff --git a/protobuf.bzl b/protobuf.bzl index 9ed7c54a..86c6f251 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -81,6 +81,7 @@ _proto_gen = rule( "deps": attr.label_list(providers = ["proto"]), "includes": attr.string_list(), "protoc": attr.label( + cfg = HOST_CFG, executable = True, single_file = True, mandatory = True, diff --git a/src/Makefile.am b/src/Makefile.am index 2985b5d8..3b66b3da 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -58,7 +58,7 @@ MAINTAINERCLEANFILES = \ nobase_include_HEADERS = \ google/protobuf/stubs/atomic_sequence_num.h \ google/protobuf/stubs/atomicops.h \ - google/protobuf/stubs/atomicops_internals_aix.h \ + google/protobuf/stubs/atomicops_internals_power.h \ google/protobuf/stubs/atomicops_internals_arm64_gcc.h \ google/protobuf/stubs/atomicops_internals_arm_gcc.h \ google/protobuf/stubs/atomicops_internals_arm_qnx.h \ diff --git a/src/google/protobuf/compiler/csharp/csharp_doc_comment.cc b/src/google/protobuf/compiler/csharp/csharp_doc_comment.cc index 9ad2cbb5..587e0222 100644 --- a/src/google/protobuf/compiler/csharp/csharp_doc_comment.cc +++ b/src/google/protobuf/compiler/csharp/csharp_doc_comment.cc @@ -56,10 +56,26 @@ void WriteDocCommentBodyImpl(io::Printer* printer, SourceLocation location) { // node of a summary element, not part of an attribute. comments = StringReplace(comments, "&", "&", true); comments = StringReplace(comments, "<", "<", true); - vector<string> lines = Split(comments, "\n"); + vector<string> lines = Split(comments, "\n", false /* skip_empty */); + // TODO: We really should work out which part to put in the summary and which to put in the remarks... + // but that needs to be part of a bigger effort to understand the markdown better anyway. printer->Print("/// <summary>\n"); + bool last_was_empty = false; + // We squash multiple blank lines down to one, and remove any trailing blank lines. We need + // to preserve the blank lines themselves, as this is relevant in the markdown. + // Note that we can't remove leading or trailing whitespace as *that's* relevant in markdown too. + // (We don't skip "just whitespace" lines, either.) for (std::vector<string>::iterator it = lines.begin(); it != lines.end(); ++it) { - printer->Print("/// $line$\n", "line", *it); + string line = *it; + if (line.empty()) { + last_was_empty = true; + } else { + if (last_was_empty) { + printer->Print("///\n"); + } + last_was_empty = false; + printer->Print("/// $line$\n", "line", *it); + } } printer->Print("/// </summary>\n"); } diff --git a/src/google/protobuf/stubs/atomicops.h b/src/google/protobuf/stubs/atomicops.h index 5fa31b0a..31e2b782 100644 --- a/src/google/protobuf/stubs/atomicops.h +++ b/src/google/protobuf/stubs/atomicops.h @@ -192,7 +192,7 @@ Atomic64 Release_Load(volatile const Atomic64* ptr); // AIX #elif defined(GOOGLE_PROTOBUF_OS_AIX) -#include <google/protobuf/stubs/atomicops_internals_aix.h> +#include <google/protobuf/stubs/atomicops_internals_power.h> // Apple. #elif defined(GOOGLE_PROTOBUF_OS_APPLE) @@ -210,6 +210,8 @@ Atomic64 Release_Load(volatile const Atomic64* ptr); #include <google/protobuf/stubs/atomicops_internals_arm_qnx.h> #elif defined(GOOGLE_PROTOBUF_ARCH_MIPS) || defined(GOOGLE_PROTOBUF_ARCH_MIPS64) #include <google/protobuf/stubs/atomicops_internals_mips_gcc.h> +#elif defined(GOOGLE_PROTOBUF_ARCH_POWER) +#include <google/protobuf/stubs/atomicops_internals_power.h> #elif defined(__native_client__) #include <google/protobuf/stubs/atomicops_internals_pnacl.h> #elif (((__GNUC__ == 4) && (__GNUC_MINOR__ >= 7)) || (__GNUC__ > 4)) diff --git a/src/google/protobuf/stubs/atomicops_internals_aix.h b/src/google/protobuf/stubs/atomicops_internals_power.h index b8a42f21..b8a42f21 100644 --- a/src/google/protobuf/stubs/atomicops_internals_aix.h +++ b/src/google/protobuf/stubs/atomicops_internals_power.h diff --git a/src/google/protobuf/stubs/platform_macros.h b/src/google/protobuf/stubs/platform_macros.h index 9e0344d8..338d22f0 100644 --- a/src/google/protobuf/stubs/platform_macros.h +++ b/src/google/protobuf/stubs/platform_macros.h @@ -70,7 +70,7 @@ #else #define GOOGLE_PROTOBUF_ARCH_32_BIT 1 #endif -#elif defined(_POWER) +#elif defined(_POWER) || defined(__powerpc64__) || defined(__PPC64__) #define GOOGLE_PROTOBUF_ARCH_POWER 1 #define GOOGLE_PROTOBUF_ARCH_64_BIT 1 #elif defined(__GNUC__) diff --git a/src/google/protobuf/util/field_comparator_test.cc b/src/google/protobuf/util/field_comparator_test.cc index 748c7d11..d3d34602 100644 --- a/src/google/protobuf/util/field_comparator_test.cc +++ b/src/google/protobuf/util/field_comparator_test.cc @@ -34,9 +34,10 @@ #include <google/protobuf/unittest.pb.h> #include <google/protobuf/descriptor.h> -#include <gtest/gtest.h> #include <google/protobuf/stubs/mathutil.h> +#include <gtest/gtest.h> + namespace google { namespace protobuf { namespace util { diff --git a/src/google/protobuf/util/internal/protostream_objectwriter.cc b/src/google/protobuf/util/internal/protostream_objectwriter.cc index 08a2fb9a..0958997c 100644 --- a/src/google/protobuf/util/internal/protostream_objectwriter.cc +++ b/src/google/protobuf/util/internal/protostream_objectwriter.cc @@ -602,7 +602,7 @@ void ProtoStreamObjectWriter::ProtoElement::TakeOneofIndex(int32 index) { bool ProtoStreamObjectWriter::ProtoElement::InsertMapKeyIfNotPresent( StringPiece map_key) { - return InsertIfNotPresent(&map_keys_, map_key); + return InsertIfNotPresent(&map_keys_, map_key.ToString()); } inline void ProtoStreamObjectWriter::InvalidName(StringPiece unknown_name, diff --git a/src/google/protobuf/util/internal/protostream_objectwriter.h b/src/google/protobuf/util/internal/protostream_objectwriter.h index 19f747fe..6e133679 100644 --- a/src/google/protobuf/util/internal/protostream_objectwriter.h +++ b/src/google/protobuf/util/internal/protostream_objectwriter.h @@ -305,7 +305,7 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public StructuredObjectWriter // Set of map keys already seen for the type_. Used to validate incoming // messages so no map key appears more than once. - hash_set<StringPiece> map_keys_; + hash_set<string> map_keys_; GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(ProtoElement); }; @@ -112,7 +112,12 @@ build_javanano_oracle7() { } internal_install_python_deps() { - sudo pip install tox + # Install tox (OS X doesn't have pip). + if [ $(uname -s) == "Darwin" ]; then + sudo easy_install tox + else + sudo pip install tox + fi # Only install Python2.6/3.x on Linux. if [ $(uname -s) == "Linux" ]; then sudo apt-get install -y python-software-properties # for apt-add-repository @@ -124,6 +129,56 @@ internal_install_python_deps() { fi } +internal_objectivec_common () { + # Make sure xctool is up to date. Adapted from + # http://docs.travis-ci.com/user/osx-ci-environment/ + # We don't use a before_install because we test multiple OSes. + brew update + brew outdated xctool || brew upgrade xctool + # Reused the build script that takes care of configuring and ensuring things + # are up to date. + objectivec/DevTools/full_mac_build.sh --core-only --skip-xcode +} + +internal_xctool_debug_and_release() { + xctool -configuration Debug "$@" + xctool -configuration Release "$@" +} + +build_objectivec_ios() { + internal_objectivec_common + # https://github.com/facebook/xctool/issues/509 - unlike xcodebuild, xctool + # doesn't support >1 destination, so we have to build first and then run the + # tests one destination at a time. + internal_xctool_debug_and_release \ + -project objectivec/ProtocolBuffers_iOS.xcodeproj \ + -scheme ProtocolBuffers \ + -sdk iphonesimulator \ + build-tests + IOS_DESTINATIONS=( + "platform=iOS Simulator,name=iPhone 4s,OS=8.1" # 32bit + "platform=iOS Simulator,name=iPhone 6,OS=9.1" # 64bit + "platform=iOS Simulator,name=iPad 2,OS=8.1" # 32bit + "platform=iOS Simulator,name=iPad Air,OS=9.1" # 64bit + ) + for i in "${IOS_DESTINATIONS[@]}" ; do + internal_xctool_debug_and_release \ + -project objectivec/ProtocolBuffers_iOS.xcodeproj \ + -scheme ProtocolBuffers \ + -sdk iphonesimulator \ + -destination "${i}" \ + run-tests + done +} + +build_objectivec_osx() { + internal_objectivec_common + internal_xctool_debug_and_release \ + -project objectivec/ProtocolBuffers_OSX.xcodeproj \ + -scheme ProtocolBuffers \ + -destination "platform=OS X,arch=x86_64" \ + test +} build_python() { internal_build_cpp @@ -189,6 +244,8 @@ Usage: $0 { cpp | javanano_jdk6 | javanano_jdk7 | javanano_oracle7 | + objectivec_ios | + objectivec_osx | python | python_cpp | ruby_19 | |