diff options
Diffstat (limited to 'conformance/conformance_test.h')
| -rw-r--r-- | conformance/conformance_test.h | 203 |
1 files changed, 122 insertions, 81 deletions
diff --git a/conformance/conformance_test.h b/conformance/conformance_test.h index 2649f8b2..d5c2f3d4 100644 --- a/conformance/conformance_test.h +++ b/conformance/conformance_test.h @@ -40,11 +40,13 @@ #include <functional> #include <string> + +#include <google/protobuf/descriptor.h> #include <google/protobuf/stubs/common.h> #include <google/protobuf/util/type_resolver.h> #include <google/protobuf/wire_format_lite.h> -#include "third_party/jsoncpp/json.h" +#include "conformance.pb.h" namespace conformance { class ConformanceRequest; @@ -60,6 +62,8 @@ class TestAllTypesProto3; namespace google { namespace protobuf { +class ConformanceTestSuite; + class ConformanceTestRunner { public: virtual ~ConformanceTestRunner() {} @@ -76,27 +80,85 @@ class ConformanceTestRunner { std::string* output) = 0; }; +// Test runner that spawns the process being tested and communicates with it +// over a pipe. +class ForkPipeRunner : public ConformanceTestRunner { + public: + static int Run(int argc, char *argv[], + ConformanceTestSuite* suite); + + private: + ForkPipeRunner(const std::string &executable) + : child_pid_(-1), executable_(executable) {} + + virtual ~ForkPipeRunner() {} + + void RunTest(const std::string& test_name, + const std::string& request, + std::string* response); + + // TODO(haberman): make this work on Windows, instead of using these + // UNIX-specific APIs. + // + // There is a platform-agnostic API in + // src/google/protobuf/compiler/subprocess.h + // + // However that API only supports sending a single message to the subprocess. + // We really want to be able to send messages and receive responses one at a + // time: + // + // 1. Spawning a new process for each test would take way too long for thousands + // of tests and subprocesses like java that can take 100ms or more to start + // up. + // + // 2. Sending all the tests in one big message and receiving all results in one + // big message would take away our visibility about which test(s) caused a + // crash or other fatal error. It would also give us only a single failure + // instead of all of them. + void SpawnTestProgram(); + + void CheckedWrite(int fd, const void *buf, size_t len); + bool TryRead(int fd, void *buf, size_t len); + void CheckedRead(int fd, void *buf, size_t len); + + int write_fd_; + int read_fd_; + pid_t child_pid_; + std::string executable_; + std::string current_test_name_; +}; + // Class representing the test suite itself. To run it, implement your own -// class derived from ConformanceTestRunner and then write code like: +// class derived from ConformanceTestRunner, class derived from +// ConformanceTestSuite and then write code like: +// +// class MyConformanceTestSuite : public ConformanceTestSuite { +// public: +// void RunSuiteImpl() { +// // INSERT ACTURAL TESTS. +// } +// }; // // class MyConformanceTestRunner : public ConformanceTestRunner { // public: +// static int Run(int argc, char *argv[], +// ConformanceTestSuite* suite); +// +// private: // virtual void RunTest(...) { // // INSERT YOUR FRAMEWORK-SPECIFIC CODE HERE. // } // }; // // int main() { -// MyConformanceTestRunner runner; -// google::protobuf::ConformanceTestSuite suite; -// -// std::string output; -// suite.RunSuite(&runner, &output); +// MyConformanceTestSuite suite; +// MyConformanceTestRunner::Run(argc, argv, &suite); // } // class ConformanceTestSuite { public: ConformanceTestSuite() : verbose_(false), enforce_recommended_(false) {} + virtual ~ConformanceTestSuite() {} void SetVerbose(bool verbose) { verbose_ = verbose; } @@ -129,7 +191,7 @@ class ConformanceTestSuite { // tests passed. bool RunSuite(ConformanceTestRunner* runner, std::string* output); - private: + protected: // Test cases are classified into a few categories: // REQUIRED: the test case must be passed for an implementation to be // interoperable with other implementations. For example, a @@ -146,7 +208,47 @@ class ConformanceTestSuite { REQUIRED = 0, RECOMMENDED = 1, }; - string ConformanceLevelToString(ConformanceLevel level); + + class ConformanceRequestSetting { + public: + ConformanceRequestSetting( + ConformanceLevel level, + conformance::WireFormat input_format, + conformance::WireFormat output_format, + conformance::TestCategory test_category, + const Message& prototype_message, + const string& test_name, const string& input); + virtual ~ConformanceRequestSetting() {} + + Message* GetTestMessage() const; + + string GetTestName() const; + + const conformance::ConformanceRequest& GetRequest() const { + return request_; + } + + const ConformanceLevel GetLevel() const { + return level_; + } + + string ConformanceLevelToString(ConformanceLevel level) const; + + protected: + virtual string InputFormatString(conformance::WireFormat format) const; + virtual string OutputFormatString(conformance::WireFormat format) const; + + private: + ConformanceLevel level_; + ::conformance::WireFormat input_format_; + ::conformance::WireFormat output_format_; + const Message& prototype_message_; + string test_name_; + conformance::ConformanceRequest request_; + }; + + bool CheckSetEmpty(const std::set<string>& set_to_check, + const std::string& write_to_file, const std::string& msg); void ReportSuccess(const std::string& test_name); void ReportFailure(const string& test_name, @@ -157,80 +259,18 @@ class ConformanceTestSuite { void ReportSkip(const string& test_name, const conformance::ConformanceRequest& request, const conformance::ConformanceResponse& response); + + void RunValidInputTest(const ConformanceRequestSetting& setting, + const string& equivalent_text_format); + void RunValidBinaryInputTest(const ConformanceRequestSetting& setting, + const string& equivalent_wire_format); + void RunTest(const std::string& test_name, const conformance::ConformanceRequest& request, conformance::ConformanceResponse* response); - void RunValidInputTest(const string& test_name, - ConformanceLevel level, - const string& input, - conformance::WireFormat input_format, - const string& equivalent_text_format, - conformance::WireFormat requested_output, - bool isProto3); - void RunValidBinaryInputTest(const string& test_name, - ConformanceLevel level, - const string& input, - conformance::WireFormat input_format, - const string& equivalent_wire_format, - conformance::WireFormat requested_output, - bool isProto3); - void RunValidJsonTest(const string& test_name, - ConformanceLevel level, - const string& input_json, - const string& equivalent_text_format); - void RunValidJsonTestWithProtobufInput( - const string& test_name, - ConformanceLevel level, - const protobuf_test_messages::proto3::TestAllTypesProto3& input, - const string& equivalent_text_format); - void RunValidProtobufTest(const string& test_name, ConformanceLevel level, - const string& input_protobuf, - const string& equivalent_text_format, - bool isProto3); - void RunValidBinaryProtobufTest(const string& test_name, - ConformanceLevel level, - const string& input_protobuf, - bool isProto3); - void RunValidProtobufTestWithMessage( - const string& test_name, ConformanceLevel level, - const Message *input, - const string& equivalent_text_format, - bool isProto3); - - typedef std::function<bool(const Json::Value&)> Validator; - void RunValidJsonTestWithValidator(const string& test_name, - ConformanceLevel level, - const string& input_json, - const Validator& validator); - void ExpectParseFailureForJson(const string& test_name, - ConformanceLevel level, - const string& input_json); - void ExpectSerializeFailureForJson(const string& test_name, - ConformanceLevel level, - const string& text_format); - void ExpectParseFailureForProtoWithProtoVersion (const string& proto, - const string& test_name, - ConformanceLevel level, - bool isProto3); - void ExpectParseFailureForProto(const std::string& proto, - const std::string& test_name, - ConformanceLevel level); - void ExpectHardParseFailureForProto(const std::string& proto, - const std::string& test_name, - ConformanceLevel level); - void TestPrematureEOFForType(google::protobuf::FieldDescriptor::Type type); - void TestIllegalTags(); - template <class MessageType> - void TestOneofMessage (MessageType &message, - bool isProto3); - template <class MessageType> - void TestUnknownMessage (MessageType &message, - bool isProto3); - void TestValidDataForType( - google::protobuf::FieldDescriptor::Type, - std::vector<std::pair<std::string, std::string>> values); - bool CheckSetEmpty(const std::set<string>& set_to_check, - const std::string& write_to_file, const std::string& msg); + + virtual void RunSuiteImpl() = 0; + ConformanceTestRunner* runner_; int successes_; int expected_failures_; @@ -256,7 +296,8 @@ class ConformanceTestSuite { // The set of tests that the testee opted out of; std::set<std::string> skipped_; - std::unique_ptr<google::protobuf::util::TypeResolver> type_resolver_; + std::unique_ptr<google::protobuf::util::TypeResolver> + type_resolver_; std::string type_url_; }; |