Add DiscardUnknownFields support for C#

By default, unknown fields are preserved when parsing. To discard
them, use a parser configured to do so:

var parser = MyMessage.Parser.WithDiscardUnknownFields(true);

5 files changed, 173 insertions, 56 deletions

diff --git a/csharp/src/Google.Protobuf.Test/UnknownFieldSetTest.cs b/csharp/src/Google.Protobuf.Test/UnknownFieldSetTest.cs
index 1edd6fba..ddf62321 100644
--- a/csharp/src/Google.Protobuf.Test/UnknownFieldSetTest.cs
+++ b/csharp/src/Google.Protobuf.Test/UnknownFieldSetTest.cs
@@ -31,6 +31,7 @@
 #endregion
 
 using System;
+using System.IO;
 using Google.Protobuf.TestProtos;
 using NUnit.Framework;
 
@@ -124,5 +125,52 @@ namespace Google.Protobuf
             Assert.AreEqual(message.CalculateSize(), otherEmptyMessage.CalculateSize());
             Assert.AreEqual(message.ToByteArray(), otherEmptyMessage.ToByteArray());
         }
+
+        [Test]
+        public void TestDiscardUnknownFields()
+        {
+            var message = SampleMessages.CreateFullTestAllTypes();
+            var goldenEmptyMessage = new TestEmptyMessage();
+            byte[] data = message.ToByteArray();
+            int fullSize = message.CalculateSize();
+
+            Action<IMessage> assertEmpty = msg =>
+            {
+                Assert.AreEqual(0, msg.CalculateSize());
+                Assert.AreEqual(goldenEmptyMessage, msg);
+            };
+
+            Action<IMessage> assertFull = msg => Assert.AreEqual(fullSize, msg.CalculateSize());
+
+            // Test the behavior of the parsers with and without discarding, both generic and non-generic.
+            MessageParser<TestEmptyMessage> retainingParser1 = TestEmptyMessage.Parser;
+            MessageParser retainingParser2 = retainingParser1;
+            MessageParser<TestEmptyMessage> discardingParser1 = retainingParser1.WithDiscardUnknownFields(true);
+            MessageParser discardingParser2 = retainingParser2.WithDiscardUnknownFields(true);
+
+            // Test parse from byte[]
+            assertFull(retainingParser1.ParseFrom(data));
+            assertFull(retainingParser2.ParseFrom(data));
+            assertEmpty(discardingParser1.ParseFrom(data));
+            assertEmpty(discardingParser2.ParseFrom(data));
+
+            // Test parse from byte[] with offset
+            assertFull(retainingParser1.ParseFrom(data, 0, data.Length));
+            assertFull(retainingParser2.ParseFrom(data, 0, data.Length));
+            assertEmpty(discardingParser1.ParseFrom(data, 0, data.Length));
+            assertEmpty(discardingParser2.ParseFrom(data, 0, data.Length));
+
+            // Test parse from CodedInputStream
+            assertFull(retainingParser1.ParseFrom(new CodedInputStream(data)));
+            assertFull(retainingParser2.ParseFrom(new CodedInputStream(data)));
+            assertEmpty(discardingParser1.ParseFrom(new CodedInputStream(data)));
+            assertEmpty(discardingParser2.ParseFrom(new CodedInputStream(data)));
+
+            // Test parse from Stream
+            assertFull(retainingParser1.ParseFrom(new MemoryStream(data)));
+            assertFull(retainingParser2.ParseFrom(new MemoryStream(data)));
+            assertEmpty(discardingParser1.ParseFrom(new MemoryStream(data)));
+            assertEmpty(discardingParser2.ParseFrom(new MemoryStream(data)));
+        }
     }
 }
diff --git a/csharp/src/Google.Protobuf/CodedInputStream.cs b/csharp/src/Google.Protobuf/CodedInputStream.cs
index 24d436c0..6bee238f 100644
--- a/csharp/src/Google.Protobuf/CodedInputStream.cs
+++ b/csharp/src/Google.Protobuf/CodedInputStream.cs
@@ -268,6 +268,11 @@ namespace Google.Protobuf
         public int RecursionLimit { get { return recursionLimit; } }

 

         /// <summary>

+        /// Internal-only property; when set to true, unknown fields will be discarded while parsing.

+        /// </summary>

+        internal bool DiscardUnknownFields { get; set; }

+

+        /// <summary>

         /// Disposes of this instance, potentially closing any underlying stream.

         /// </summary>

         /// <remarks>

diff --git a/csharp/src/Google.Protobuf/MessageExtensions.cs b/csharp/src/Google.Protobuf/MessageExtensions.cs
index 9dbc49d6..62181eb9 100644
--- a/csharp/src/Google.Protobuf/MessageExtensions.cs
+++ b/csharp/src/Google.Protobuf/MessageExtensions.cs
@@ -44,14 +44,8 @@ namespace Google.Protobuf
         /// </summary>
         /// <param name="message">The message to merge the data into.</param>
         /// <param name="data">The data to merge, which must be protobuf-encoded binary data.</param>
-        public static void MergeFrom(this IMessage message, byte[] data)
-        {
-            ProtoPreconditions.CheckNotNull(message, "message");
-            ProtoPreconditions.CheckNotNull(data, "data");
-            CodedInputStream input = new CodedInputStream(data);
-            message.MergeFrom(input);
-            input.CheckReadEndOfStreamTag();
-        }
+        public static void MergeFrom(this IMessage message, byte[] data) =>
+            MergeFrom(message, data, false);
 
         /// <summary>
         /// Merges data from the given byte array slice into an existing message.
@@ -60,42 +54,24 @@ namespace Google.Protobuf
         /// <param name="data">The data containing the slice to merge, which must be protobuf-encoded binary data.</param>
         /// <param name="offset">The offset of the slice to merge.</param>
         /// <param name="length">The length of the slice to merge.</param>
-        public static void MergeFrom(this IMessage message, byte[] data, int offset, int length)
-        {
-            ProtoPreconditions.CheckNotNull(message, "message");
-            ProtoPreconditions.CheckNotNull(data, "data");
-            CodedInputStream input = new CodedInputStream(data, offset, length);
-            message.MergeFrom(input);
-            input.CheckReadEndOfStreamTag();
-        }
+        public static void MergeFrom(this IMessage message, byte[] data, int offset, int length) =>
+            MergeFrom(message, data, offset, length, false);
 
         /// <summary>
         /// Merges data from the given byte string into an existing message.
         /// </summary>
         /// <param name="message">The message to merge the data into.</param>
         /// <param name="data">The data to merge, which must be protobuf-encoded binary data.</param>
-        public static void MergeFrom(this IMessage message, ByteString data)
-        {
-            ProtoPreconditions.CheckNotNull(message, "message");
-            ProtoPreconditions.CheckNotNull(data, "data");
-            CodedInputStream input = data.CreateCodedInput();
-            message.MergeFrom(input);
-            input.CheckReadEndOfStreamTag();
-        }
+        public static void MergeFrom(this IMessage message, ByteString data) =>
+            MergeFrom(message, data, false);
 
         /// <summary>
         /// Merges data from the given stream into an existing message.
         /// </summary>
         /// <param name="message">The message to merge the data into.</param>
         /// <param name="input">Stream containing the data to merge, which must be protobuf-encoded binary data.</param>
-        public static void MergeFrom(this IMessage message, Stream input)
-        {
-            ProtoPreconditions.CheckNotNull(message, "message");
-            ProtoPreconditions.CheckNotNull(input, "input");
-            CodedInputStream codedInput = new CodedInputStream(input);
-            message.MergeFrom(codedInput);
-            codedInput.CheckReadEndOfStreamTag();
-        }
+        public static void MergeFrom(this IMessage message, Stream input) =>
+            MergeFrom(message, input, false);
 
         /// <summary>
         /// Merges length-delimited data from the given stream into an existing message.
@@ -106,14 +82,8 @@ namespace Google.Protobuf
         /// </remarks>
         /// <param name="message">The message to merge the data into.</param>
         /// <param name="input">Stream containing the data to merge, which must be protobuf-encoded binary data.</param>
-        public static void MergeDelimitedFrom(this IMessage message, Stream input)
-        {
-            ProtoPreconditions.CheckNotNull(message, "message");
-            ProtoPreconditions.CheckNotNull(input, "input");
-            int size = (int) CodedInputStream.ReadRawVarint32(input);
-            Stream limitedStream = new LimitedInputStream(input, size);
-            message.MergeFrom(limitedStream);
-        }
+        public static void MergeDelimitedFrom(this IMessage message, Stream input) =>
+            MergeDelimitedFrom(message, input, false);
 
         /// <summary>
         /// Converts the given message into a byte array in protobuf encoding.
@@ -168,6 +138,56 @@ namespace Google.Protobuf
         {
             ProtoPreconditions.CheckNotNull(message, "message");
             return ByteString.AttachBytes(message.ToByteArray());
-        }        
+        }
+
+        // Implementations allowing unknown fields to be discarded.
+        internal static void MergeFrom(this IMessage message, byte[] data, bool discardUnknownFields)
+        {
+            ProtoPreconditions.CheckNotNull(message, "message");
+            ProtoPreconditions.CheckNotNull(data, "data");
+            CodedInputStream input = new CodedInputStream(data);
+            input.DiscardUnknownFields = discardUnknownFields;
+            message.MergeFrom(input);
+            input.CheckReadEndOfStreamTag();
+        }
+
+        internal static void MergeFrom(this IMessage message, byte[] data, int offset, int length, bool discardUnknownFields)
+        {
+            ProtoPreconditions.CheckNotNull(message, "message");
+            ProtoPreconditions.CheckNotNull(data, "data");
+            CodedInputStream input = new CodedInputStream(data, offset, length);
+            input.DiscardUnknownFields = discardUnknownFields;
+            message.MergeFrom(input);
+            input.CheckReadEndOfStreamTag();
+        }
+
+        internal static void MergeFrom(this IMessage message, ByteString data, bool discardUnknownFields)
+        {
+            ProtoPreconditions.CheckNotNull(message, "message");
+            ProtoPreconditions.CheckNotNull(data, "data");
+            CodedInputStream input = data.CreateCodedInput();
+            input.DiscardUnknownFields = discardUnknownFields;
+            message.MergeFrom(input);
+            input.CheckReadEndOfStreamTag();
+        }
+
+        internal static void MergeFrom(this IMessage message, Stream input, bool discardUnknownFields)
+        {
+            ProtoPreconditions.CheckNotNull(message, "message");
+            ProtoPreconditions.CheckNotNull(input, "input");
+            CodedInputStream codedInput = new CodedInputStream(input);
+            codedInput.DiscardUnknownFields = discardUnknownFields;
+            message.MergeFrom(codedInput);
+            codedInput.CheckReadEndOfStreamTag();
+        }
+
+        internal static void MergeDelimitedFrom(this IMessage message, Stream input, bool discardUnknownFields)
+        {
+            ProtoPreconditions.CheckNotNull(message, "message");
+            ProtoPreconditions.CheckNotNull(input, "input");
+            int size = (int) CodedInputStream.ReadRawVarint32(input);
+            Stream limitedStream = new LimitedInputStream(input, size);
+            MergeFrom(message, limitedStream, discardUnknownFields);
+        }
     }
 }
diff --git a/csharp/src/Google.Protobuf/MessageParser.cs b/csharp/src/Google.Protobuf/MessageParser.cs
index 66d44135..4d35554a 100644
--- a/csharp/src/Google.Protobuf/MessageParser.cs
+++ b/csharp/src/Google.Protobuf/MessageParser.cs
@@ -42,10 +42,13 @@ namespace Google.Protobuf
     public class MessageParser
     {
         private Func<IMessage> factory;
+        // TODO: When we use a C# 7.1 compiler, make this private protected.
+        internal bool DiscardUnknownFields { get; }
 
-        internal MessageParser(Func<IMessage> factory)
+        internal MessageParser(Func<IMessage> factory, bool discardUnknownFields)
         {
             this.factory = factory;
+            DiscardUnknownFields = discardUnknownFields;
         }
 
         /// <summary>
@@ -65,7 +68,7 @@ namespace Google.Protobuf
         public IMessage ParseFrom(byte[] data)
         {
             IMessage message = factory();
-            message.MergeFrom(data);
+            message.MergeFrom(data, DiscardUnknownFields);
             return message;
         }
 
@@ -79,7 +82,7 @@ namespace Google.Protobuf
         public IMessage ParseFrom(byte[] data, int offset, int length)
         {
             IMessage message = factory();
-            message.MergeFrom(data, offset, length);
+            message.MergeFrom(data, offset, length, DiscardUnknownFields);
             return message;
         }
 
@@ -91,7 +94,7 @@ namespace Google.Protobuf
         public IMessage ParseFrom(ByteString data)
         {
             IMessage message = factory();
-            message.MergeFrom(data);
+            message.MergeFrom(data, DiscardUnknownFields);
             return message;
         }
 
@@ -103,7 +106,7 @@ namespace Google.Protobuf
         public IMessage ParseFrom(Stream input)
         {
             IMessage message = factory();
-            message.MergeFrom(input);
+            message.MergeFrom(input, DiscardUnknownFields);
             return message;
         }
 
@@ -119,7 +122,7 @@ namespace Google.Protobuf
         public IMessage ParseDelimitedFrom(Stream input)
         {
             IMessage message = factory();
-            message.MergeDelimitedFrom(input);
+            message.MergeDelimitedFrom(input, DiscardUnknownFields);
             return message;
         }
 
@@ -131,7 +134,7 @@ namespace Google.Protobuf
         public IMessage ParseFrom(CodedInputStream input)
         {
             IMessage message = factory();
-            message.MergeFrom(input);
+            MergeFrom(message, input);
             return message;
         }
 
@@ -148,6 +151,29 @@ namespace Google.Protobuf
             JsonParser.Default.Merge(message, json);
             return message;
         }
+
+        // TODO: When we're using a C# 7.1 compiler, make this private protected.
+        internal void MergeFrom(IMessage message, CodedInputStream codedInput)
+        {
+            bool originalDiscard = codedInput.DiscardUnknownFields;
+            try
+            {
+                codedInput.DiscardUnknownFields = DiscardUnknownFields;
+                message.MergeFrom(codedInput);
+            }
+            finally
+            {
+                codedInput.DiscardUnknownFields = originalDiscard;
+            }
+        }
+
+        /// <summary>
+        /// Creates a new message parser which optionally discards unknown fields when parsing.
+        /// </summary>
+        /// <param name="discardUnknownFields">Whether or not to discard unknown fields when parsing.</param>
+        /// <returns>A newly configured message parser.</returns>
+        public MessageParser WithDiscardUnknownFields(bool discardUnknownFields) =>
+            new MessageParser(factory, discardUnknownFields);
     }
 
     /// <summary>
@@ -182,7 +208,11 @@ namespace Google.Protobuf
         /// to require a parameterless constructor: delegates are significantly faster to execute.
         /// </remarks>
         /// <param name="factory">Function to invoke when a new, empty message is required.</param>
-        public MessageParser(Func<T> factory) : base(() => factory())
+        public MessageParser(Func<T> factory) : this(factory, false)
+        {
+        }
+
+        internal MessageParser(Func<T> factory, bool discardUnknownFields) : base(() => factory(), discardUnknownFields)
         {
             this.factory = factory;
         }
@@ -204,7 +234,7 @@ namespace Google.Protobuf
         public new T ParseFrom(byte[] data)
         {
             T message = factory();
-            message.MergeFrom(data);
+            message.MergeFrom(data, DiscardUnknownFields);
             return message;
         }
 
@@ -218,7 +248,7 @@ namespace Google.Protobuf
         public new T ParseFrom(byte[] data, int offset, int length)
         {
             T message = factory();
-            message.MergeFrom(data, offset, length);
+            message.MergeFrom(data, offset, length, DiscardUnknownFields);
             return message;
         }
 
@@ -230,7 +260,7 @@ namespace Google.Protobuf
         public new T ParseFrom(ByteString data)
         {
             T message = factory();
-            message.MergeFrom(data);
+            message.MergeFrom(data, DiscardUnknownFields);
             return message;
         }
 
@@ -242,7 +272,7 @@ namespace Google.Protobuf
         public new T ParseFrom(Stream input)
         {
             T message = factory();
-            message.MergeFrom(input);
+            message.MergeFrom(input, DiscardUnknownFields);
             return message;
         }
 
@@ -258,7 +288,7 @@ namespace Google.Protobuf
         public new T ParseDelimitedFrom(Stream input)
         {
             T message = factory();
-            message.MergeDelimitedFrom(input);
+            message.MergeDelimitedFrom(input, DiscardUnknownFields);
             return message;
         }
 
@@ -270,7 +300,7 @@ namespace Google.Protobuf
         public new T ParseFrom(CodedInputStream input)
         {
             T message = factory();
-            message.MergeFrom(input);
+            MergeFrom(message, input);
             return message;
         }
 
@@ -287,5 +317,13 @@ namespace Google.Protobuf
             JsonParser.Default.Merge(message, json);
             return message;
         }
+
+        /// <summary>
+        /// Creates a new message parser which optionally discards unknown fields when parsing.
+        /// </summary>
+        /// <param name="discardUnknownFields">Whether or not to discard unknown fields when parsing.</param>
+        /// <returns>A newly configured message parser.</returns>
+        public new MessageParser<T> WithDiscardUnknownFields(bool discardUnknownFields) =>
+            new MessageParser<T>(factory, discardUnknownFields);
     }
 }
diff --git a/csharp/src/Google.Protobuf/UnknownFieldSet.cs b/csharp/src/Google.Protobuf/UnknownFieldSet.cs
index b43f4774..6404c3c0 100644
--- a/csharp/src/Google.Protobuf/UnknownFieldSet.cs
+++ b/csharp/src/Google.Protobuf/UnknownFieldSet.cs
@@ -230,7 +230,8 @@ namespace Google.Protobuf
         /// <summary>
         /// Create a new UnknownFieldSet if unknownFields is null.
         /// Parse a single field from <paramref name="input"/> and merge it
-        /// into unknownFields.
+        /// into unknownFields. If <paramref name="input"/> is configured to discard unknown fields,
+        /// <paramref name="unknownFields"/> will be returned as-is and the field will be skipped.
         /// </summary>
         /// <param name="unknownFields">The UnknownFieldSet which need to be merged</param>
         /// <param name="input">The coded input stream containing the field</param>
@@ -238,6 +239,11 @@ namespace Google.Protobuf
         public static UnknownFieldSet MergeFieldFrom(UnknownFieldSet unknownFields,
                                                      CodedInputStream input)
         {
+            if (input.DiscardUnknownFields)
+            {
+                input.SkipLastField();
+                return unknownFields;
+            }
             if (unknownFields == null)
             {
                 unknownFields = new UnknownFieldSet();