Give a specific category to each test. (#4965)

* Give a unique category to each test.

This change introduce a TestCategory enum to ConformanceRequest. Existing tests
are divided into three categories: binary format test, json format test and json
format (ignore unknown when parsing) test. For the previous two categories, there
is no change to existing testee programs. For tests with the last category, testee programs
should either enable ignoring unknown field during json parsing or skip the test.

* Fix python test

* Fix java

* Fix csharp

* Update document

* Update csharp generated code

13 files changed, 55 insertions, 53 deletions

diff --git a/conformance/ConformanceJava.java b/conformance/ConformanceJava.java
index 596d113a..dcdc27e3 100644
--- a/conformance/ConformanceJava.java
+++ b/conformance/ConformanceJava.java
@@ -235,8 +235,12 @@ class ConformanceJava {
         try {
           TestMessagesProto3.TestAllTypesProto3.Builder builder = 
               TestMessagesProto3.TestAllTypesProto3.newBuilder();
-          JsonFormat.parser().usingTypeRegistry(typeRegistry)
-              .merge(request.getJsonPayload(), builder);
+          JsonFormat.Parser parser = JsonFormat.parser().usingTypeRegistry(typeRegistry);
+          if (request.getTestCategory()
+              == Conformance.TestCategory.JSON_IGNORE_UNKNOWN_PARSING_TEST) {
+            parser = parser.ignoringUnknownFields();
+          }
+          parser.merge(request.getJsonPayload(), builder);
           testMessage = builder.build();
         } catch (InvalidProtocolBufferException e) {
           return Conformance.ConformanceResponse.newBuilder().setParseError(e.getMessage()).build();
diff --git a/conformance/conformance.proto b/conformance/conformance.proto
index 897e7b38..d838e578 100644
--- a/conformance/conformance.proto
+++ b/conformance/conformance.proto
@@ -57,6 +57,17 @@ enum WireFormat {
   JSON = 2;
 }
 
+enum TestCategory {
+  BINARY_TEST = 0;  // Test binary wire format.
+  JSON_TEST = 1;  // Test json wire format.
+  // Similar to JSON_TEST. However, during parsing json, testee should ignore
+  // unknown fields. This feature is optional. Each implementation can descide
+  // whether to support it.  See
+  // https://developers.google.com/protocol-buffers/docs/proto3#json_options
+  // for more detail.
+  JSON_IGNORE_UNKNOWN_PARSING_TEST = 2;  
+}
+
 // Represents a single test case's input.  The testee should:
 //
 //   1. parse this proto (which should always succeed)
@@ -83,7 +94,10 @@ message ConformanceRequest {
   // protobuf_test_messages.proto2.TestAllTypesProto2.
   string message_type = 4;
 
-  bool ignore_unknown_json = 5;
+  // Each test is given a specific test category. Some category may need spedific
+  // support in testee programs. Refer to the defintion of TestCategory for
+  // more information.
+  TestCategory test_category = 5;
 }
 
 // Represents a single test case's output.
diff --git a/conformance/conformance_cpp.cc b/conformance/conformance_cpp.cc
index 97ae1a7a..ac2f6dea 100644
--- a/conformance/conformance_cpp.cc
+++ b/conformance/conformance_cpp.cc
@@ -46,6 +46,7 @@ using google::protobuf::DescriptorPool;
 using google::protobuf::Message;
 using google::protobuf::MessageFactory;
 using google::protobuf::util::BinaryToJsonString;
+using google::protobuf::util::JsonParseOptions;
 using google::protobuf::util::JsonToBinaryString;
 using google::protobuf::util::NewTypeResolverForDescriptorPool;
 using google::protobuf::util::Status;
@@ -112,8 +113,13 @@ void DoTest(const ConformanceRequest& request, ConformanceResponse* response) {
 
     case ConformanceRequest::kJsonPayload: {
       string proto_binary;
+      JsonParseOptions options;
+      options.ignore_unknown_fields =
+          (request.test_category() ==
+              conformance::JSON_IGNORE_UNKNOWN_PARSING_TEST);
       Status status = JsonToBinaryString(type_resolver, *type_url,
-                                         request.json_payload(), &proto_binary);
+                                         request.json_payload(), &proto_binary,
+                                         options);
       if (!status.ok()) {
         response->set_parse_error(string("Parse error: ") +
                                   status.error_message().as_string());
diff --git a/conformance/conformance_php.php b/conformance/conformance_php.php
index 65483e27..799cc3e3 100755
--- a/conformance/conformance_php.php
+++ b/conformance/conformance_php.php
@@ -3,6 +3,7 @@
 require_once("Conformance/WireFormat.php");
 require_once("Conformance/ConformanceResponse.php");
 require_once("Conformance/ConformanceRequest.php");
+require_once("Conformance/TestCategory.php");
 require_once("Protobuf_test_messages/Proto3/ForeignMessage.php");
 require_once("Protobuf_test_messages/Proto3/ForeignEnum.php");
 require_once("Protobuf_test_messages/Proto3/TestAllTypesProto3.php");
@@ -12,6 +13,7 @@ require_once("Protobuf_test_messages/Proto3/TestAllTypesProto3/NestedEnum.php");
 require_once("GPBMetadata/Conformance.php");
 require_once("GPBMetadata/Google/Protobuf/TestMessagesProto3.php");
 
+use  \Conformance\TestCategory;
 use  \Conformance\WireFormat;
 
 if (!ini_get("date.timezone")) {
@@ -39,7 +41,9 @@ function doTest($request)
         trigger_error("Protobuf request doesn't have specific payload type", E_USER_ERROR);
       }
     } elseif ($request->getPayload() == "json_payload") {
-      $ignore_json_unknown = $request->getIgnoreUnknownJson();
+      $ignore_json_unknown =
+          ($request->getTestCategory() ==
+              TestCategory::JSON_IGNORE_UNKNOWN_PARSING_TEST);
       try {
           $test_message->mergeFromJsonString($request->getJsonPayload(),
                                              $ignore_json_unknown);
diff --git a/conformance/conformance_python.py b/conformance/conformance_python.py
index c5ba2467..876642bc 100755
--- a/conformance/conformance_python.py
+++ b/conformance/conformance_python.py
@@ -78,7 +78,11 @@ def do_test(request):
       
     elif request.WhichOneof('payload') == 'json_payload':
       try:
-        json_format.Parse(request.json_payload, test_message)
+        ignore_unknown_fields = \
+            request.test_category == \
+                conformance_pb2.JSON_IGNORE_UNKNOWN_PARSING_TEST
+        json_format.Parse(request.json_payload, test_message,
+                          ignore_unknown_fields)
       except Exception as e:
         response.parse_error = str(e)
         return response
diff --git a/conformance/conformance_test.cc b/conformance/conformance_test.cc
index d9e88ce5..74342b68 100644
--- a/conformance/conformance_test.cc
+++ b/conformance/conformance_test.cc
@@ -193,10 +193,11 @@ namespace protobuf {
 
 ConformanceTestSuite::ConformanceRequestSetting::ConformanceRequestSetting(
     ConformanceLevel level, conformance::WireFormat input_format,
-    conformance::WireFormat output_format, bool is_proto3,
+    conformance::WireFormat output_format,
+    conformance::TestCategory test_category,
+    bool is_proto3,
     const string& test_name, const string& input)
-    : level_(level), input_format_(input_format),
-      output_format_(output_format), is_proto3_(is_proto3) {
+    : level_(level), is_proto3_(is_proto3) {
   auto newTestMessage = [&is_proto3]() {
     Message* newMessage;
     if (is_proto3) {
@@ -243,6 +244,8 @@ ConformanceTestSuite::ConformanceRequestSetting::ConformanceRequestSetting(
       GOOGLE_LOG(FATAL) << "Unspecified output format";
   }
 
+  request_.set_test_category(test_category);
+
   test_name_ = ConformanceLevelToString(level) + rname +
                input_format_string + test_name +
                output_format_string;
@@ -465,6 +468,7 @@ void ConformanceTestSuite::ExpectParseFailureForProtoWithProtoVersion (
   ConformanceRequest request;
   ConformanceResponse response;
   request.set_protobuf_payload(proto);
+  request.set_test_category(conformance::BINARY_TEST);
   if (isProto3) {
     request.set_message_type("protobuf_test_messages.proto3.TestAllTypesProto3");
   } else {
@@ -511,11 +515,13 @@ void ConformanceTestSuite::RunValidJsonTest(
     const string& equivalent_text_format) {
   ConformanceRequestSetting setting1(
       level, conformance::JSON, conformance::PROTOBUF,
+      conformance::JSON_TEST,
       true, test_name, input_json);
   RunValidInputTest(setting1, equivalent_text_format);
 
   ConformanceRequestSetting setting2(
       level, conformance::JSON, conformance::JSON,
+      conformance::JSON_TEST,
       true, test_name, input_json);
   RunValidInputTest(setting2, equivalent_text_format);
 }
@@ -525,6 +531,7 @@ void ConformanceTestSuite::RunValidJsonTestWithProtobufInput(
     const string& equivalent_text_format) {
   ConformanceRequestSetting setting(
       level, conformance::PROTOBUF, conformance::JSON,
+      conformance::JSON_TEST,
       true, test_name, input.SerializeAsString());
   RunValidInputTest(setting, equivalent_text_format);
 }
@@ -534,8 +541,8 @@ void ConformanceTestSuite::RunValidJsonIgnoreUnknownTest(
     const string& equivalent_text_format) {
   ConformanceRequestSetting setting(
       level, conformance::JSON, conformance::PROTOBUF,
+      conformance::JSON_IGNORE_UNKNOWN_PARSING_TEST,
       true, test_name, input_json);
-  setting.SetIgnoreUnknownJson(true);
   RunValidInputTest(setting, equivalent_text_format);
 }
 
@@ -545,12 +552,14 @@ void ConformanceTestSuite::RunValidProtobufTest(
     bool isProto3) {
   ConformanceRequestSetting setting1(
       level, conformance::PROTOBUF, conformance::PROTOBUF,
+      conformance::BINARY_TEST,
       isProto3, test_name, input_protobuf);
   RunValidInputTest(setting1, equivalent_text_format);
 
   if (isProto3) {
     ConformanceRequestSetting setting2(
         level, conformance::PROTOBUF, conformance::JSON,
+        conformance::BINARY_TEST,
         true, test_name, input_protobuf);
     RunValidInputTest(setting2, equivalent_text_format);
   }
@@ -561,6 +570,7 @@ void ConformanceTestSuite::RunValidBinaryProtobufTest(
     const string& input_protobuf, bool isProto3) {
   ConformanceRequestSetting setting(
       level, conformance::PROTOBUF, conformance::PROTOBUF,
+      conformance::BINARY_TEST,
       isProto3, test_name, input_protobuf);
   RunValidBinaryInputTest(setting, input_protobuf);
 }
diff --git a/conformance/conformance_test.h b/conformance/conformance_test.h
index 685f67fb..2044896f 100644
--- a/conformance/conformance_test.h
+++ b/conformance/conformance_test.h
@@ -152,7 +152,9 @@ class ConformanceTestSuite {
    public:
     ConformanceRequestSetting(
         ConformanceLevel level, conformance::WireFormat input_format,
-        conformance::WireFormat output_format, bool is_proto3,
+        conformance::WireFormat output_format,
+        conformance::TestCategory test_category,
+        bool is_proto3,
         const string& test_name, const string& input);
 
    Message* GetTestMessage() const;
@@ -169,14 +171,8 @@ class ConformanceTestSuite {
      return level_;
    }
 
-   void SetIgnoreUnknownJson(bool ignore_unknown_json) {
-     request_.set_ignore_unknown_json(ignore_unknown_json);
-   }
-  
    private:
     ConformanceLevel level_;
-    conformance::WireFormat input_format_;
-    conformance::WireFormat output_format_;
     bool is_proto3_;
     string test_name_;
     conformance::ConformanceRequest request_;
diff --git a/conformance/failure_list_cpp.txt b/conformance/failure_list_cpp.txt
index ea8e8473..752fbb5d 100644
--- a/conformance/failure_list_cpp.txt
+++ b/conformance/failure_list_cpp.txt
@@ -54,9 +54,3 @@ Required.Proto2.ProtobufInput.PrematureEofInPackedField.SINT32
 Required.Proto2.ProtobufInput.PrematureEofInPackedField.SINT64
 Required.Proto2.ProtobufInput.PrematureEofInPackedField.UINT32
 Required.Proto2.ProtobufInput.PrematureEofInPackedField.UINT64
-Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput
diff --git a/conformance/failure_list_csharp.txt b/conformance/failure_list_csharp.txt
index 69c723b5..2a20aa78 100644
--- a/conformance/failure_list_csharp.txt
+++ b/conformance/failure_list_csharp.txt
@@ -1,8 +1,2 @@
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput
diff --git a/conformance/failure_list_java.txt b/conformance/failure_list_java.txt
index 6f085c66..dc1f9ba5 100644
--- a/conformance/failure_list_java.txt
+++ b/conformance/failure_list_java.txt
@@ -45,9 +45,3 @@ Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValu
 Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE
 Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE
 Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE
-Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput
diff --git a/conformance/failure_list_python-post26.txt b/conformance/failure_list_python-post26.txt
index 60b1146e..19d99b04 100644
--- a/conformance/failure_list_python-post26.txt
+++ b/conformance/failure_list_python-post26.txt
@@ -1,8 +1,2 @@
 JsonInput.StringFieldSurrogateInWrongOrder
 JsonInput.StringFieldUnpairedHighSurrogate
-Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput
diff --git a/conformance/failure_list_python.txt b/conformance/failure_list_python.txt
index 5f01b53b..e3ce7af7 100644
--- a/conformance/failure_list_python.txt
+++ b/conformance/failure_list_python.txt
@@ -19,9 +19,3 @@ Required.Proto3.ProtobufInput.IllegalZeroFieldNum_Case_0
 Required.Proto3.ProtobufInput.IllegalZeroFieldNum_Case_1
 Required.Proto3.ProtobufInput.IllegalZeroFieldNum_Case_2
 Required.Proto3.ProtobufInput.IllegalZeroFieldNum_Case_3
-Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput
diff --git a/conformance/failure_list_python_cpp.txt b/conformance/failure_list_python_cpp.txt
index f80517d9..a498ad1a 100644
--- a/conformance/failure_list_python_cpp.txt
+++ b/conformance/failure_list_python_cpp.txt
@@ -52,9 +52,3 @@ Required.Proto2.ProtobufInput.PrematureEofInPackedField.SINT32
 Required.Proto2.ProtobufInput.PrematureEofInPackedField.SINT64
 Required.Proto2.ProtobufInput.PrematureEofInPackedField.UINT32
 Required.Proto2.ProtobufInput.PrematureEofInPackedField.UINT64
-Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput
-Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput