Updated C# tests and code for TextFormat to match Java.

3 files changed, 85 insertions, 9 deletions

diff --git a/csharp/ProtocolBuffers.Test/TextFormatTest.cs b/csharp/ProtocolBuffers.Test/TextFormatTest.cs
index a8cd3557..8d30b96c 100644
--- a/csharp/ProtocolBuffers.Test/TextFormatTest.cs
+++ b/csharp/ProtocolBuffers.Test/TextFormatTest.cs
@@ -219,6 +219,37 @@ namespace Google.ProtocolBuffers {
     }
 
     [Test]
+    public void ParseCompatibility() {
+      string original = "repeated_float: inf\n" +
+                        "repeated_float: -inf\n" +
+                        "repeated_float: nan\n" +
+                        "repeated_float: inff\n" +
+                        "repeated_float: -inff\n" +
+                        "repeated_float: nanf\n" +
+                        "repeated_float: 1.0f\n" +
+                        "repeated_float: infinityf\n" +
+                        "repeated_float: -Infinityf\n" +
+                        "repeated_double: infinity\n" +
+                        "repeated_double: -infinity\n" +
+                        "repeated_double: nan\n";
+      string canonical = "repeated_float: Infinity\n" +
+                          "repeated_float: -Infinity\n" +
+                          "repeated_float: NaN\n" +
+                          "repeated_float: Infinity\n" +
+                          "repeated_float: -Infinity\n" +
+                          "repeated_float: NaN\n" +
+                          "repeated_float: 1\n" + // Java has 1.0; this is fine
+                          "repeated_float: Infinity\n" +
+                          "repeated_float: -Infinity\n" +
+                          "repeated_double: Infinity\n" +
+                          "repeated_double: -Infinity\n" +
+                          "repeated_double: NaN\n";
+      TestAllTypes.Builder builder = TestAllTypes.CreateBuilder();
+      TextFormat.Merge(original, builder);
+      Assert.AreEqual(canonical, builder.Build().ToString());
+    }
+
+    [Test]
     public void ParseExotic() {
       TestAllTypes.Builder builder = TestAllTypes.CreateBuilder();
       TextFormat.Merge(ExoticText, builder);
@@ -259,6 +290,19 @@ namespace Google.ProtocolBuffers {
       Assert.AreEqual(1, builder.OptionalGroup.A);
     }
 
+    [Test]
+    public void ParseComment() {
+      TestAllTypes.Builder builder = TestAllTypes.CreateBuilder();
+      TextFormat.Merge(
+        "# this is a comment\n" +
+        "optional_int32: 1  # another comment\n" +
+        "optional_int64: 2\n" +
+        "# EOF comment", builder);
+      Assert.AreEqual(1, builder.OptionalInt32);
+      Assert.AreEqual(2, builder.OptionalInt64);
+    }
+
+
     private static void AssertParseError(string error, string text) {
       TestAllTypes.Builder builder = TestAllTypes.CreateBuilder();
       try {
diff --git a/csharp/ProtocolBuffers/TextFormat.cs b/csharp/ProtocolBuffers/TextFormat.cs
index 36c8c022..14f1ec73 100644
--- a/csharp/ProtocolBuffers/TextFormat.cs
+++ b/csharp/ProtocolBuffers/TextFormat.cs
@@ -40,8 +40,6 @@ namespace Google.ProtocolBuffers {
     /// <summary>
     /// Outputs a textual representation of <paramref name="fields" /> to <paramref name="output"/>.
     /// </summary>
-    /// <param name="fields"></param>
-    /// <param name="output"></param>
     public static void Print(UnknownFieldSet fields, TextWriter output) {
       TextGenerator generator = new TextGenerator(output);
       PrintUnknownFields(fields, generator);
@@ -564,7 +562,7 @@ namespace Google.ProtocolBuffers {
             break;
 
           case FieldType.Float:
-            value = tokenizer.consumeFloat();
+            value = tokenizer.ConsumeFloat();
             break;
 
           case FieldType.Double:
diff --git a/csharp/ProtocolBuffers/TextTokenizer.cs b/csharp/ProtocolBuffers/TextTokenizer.cs
index 81c83c7d..d53ae596 100644
--- a/csharp/ProtocolBuffers/TextTokenizer.cs
+++ b/csharp/ProtocolBuffers/TextTokenizer.cs
@@ -53,13 +53,18 @@ namespace Google.ProtocolBuffers {
     /// </summary>
     private int previousColumn = 0;
 
-    private static Regex WhitespaceAndCommentPattern = new Regex("\\G(\\s|(#[^\\\n]*\\n))+", RegexOptions.Compiled);
-    private static Regex TokenPattern = new Regex(
+    private static readonly Regex WhitespaceAndCommentPattern = new Regex("\\G(\\s|(#.*$))+", 
+        RegexOptions.Compiled | RegexOptions.Multiline);
+    private static readonly Regex TokenPattern = new Regex(
       "\\G[a-zA-Z_][0-9a-zA-Z_+-]*|" +              // an identifier
       "\\G[0-9+-][0-9a-zA-Z_.+-]*|" +                  // a number
-      "\\G\"([^\"\\\n\\\\]|\\\\[^\\\n])*(\"|\\\\?$)|" +    // a double-quoted string
-      "\\G\'([^\"\\\n\\\\]|\\\\[^\\\n])*(\'|\\\\?$)",      // a single-quoted string
-      RegexOptions.Compiled);
+      "\\G\"([^\"\\\n\\\\]|\\\\.)*(\"|\\\\?$)|" +    // a double-quoted string
+      "\\G\'([^\"\\\n\\\\]|\\\\.)*(\'|\\\\?$)",      // a single-quoted string
+      RegexOptions.Compiled | RegexOptions.Multiline);
+
+    private static readonly Regex DoubleInfinity = new Regex("^-?inf(inity)?$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
+    private static readonly Regex FloatInfinity = new Regex("^-?inf(inity)?f?$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
+    private static readonly Regex FloatNan = new Regex("^nanf?$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
 
     /** Construct a tokenizer that parses tokens from the given text. */
     public TextTokenizer(string text) {
@@ -243,6 +248,18 @@ namespace Google.ProtocolBuffers {
     /// Otherwise, throw a FormatException.
     /// </summary>
     public double ConsumeDouble() {
+      // We need to parse infinity and nan separately because
+      // double.Parse() does not accept "inf", "infinity", or "nan".
+      if (DoubleInfinity.IsMatch(currentToken)) {
+        bool negative = currentToken.StartsWith("-");
+        NextToken();
+        return negative ? double.NegativeInfinity : double.PositiveInfinity;
+      }
+      if (currentToken.Equals("nan", StringComparison.InvariantCultureIgnoreCase)) {
+        NextToken();
+        return Double.NaN;
+      }
+
       try {
         double result = double.Parse(currentToken, CultureInfo.InvariantCulture);
         NextToken();
@@ -258,7 +275,24 @@ namespace Google.ProtocolBuffers {
     /// If the next token is a float, consume it and return its value.
     /// Otherwise, throw a FormatException.
     /// </summary>
-    public float consumeFloat() {
+    public float ConsumeFloat() {
+
+      // We need to parse infinity and nan separately because
+      // Float.parseFloat() does not accept "inf", "infinity", or "nan".
+      if (FloatInfinity.IsMatch(currentToken)) {
+        bool negative = currentToken.StartsWith("-");
+        NextToken();
+        return negative ? float.NegativeInfinity : float.PositiveInfinity;
+      }
+      if (FloatNan.IsMatch(currentToken)) {
+        NextToken();
+        return float.NaN;
+      }
+
+      if (currentToken.EndsWith("f")) {
+        currentToken = currentToken.TrimEnd('f');
+      }
+
       try {
         float result = float.Parse(currentToken, CultureInfo.InvariantCulture);
         NextToken();