diff options
158 files changed, 2488 insertions, 1122 deletions
@@ -43,12 +43,12 @@ COPTS = select({ "//conditions:default": [ "-DHAVE_PTHREAD", "-Wall", - "-Wwrite-strings", "-Woverloaded-virtual", "-Wno-sign-compare", "-Wno-unused-function", # Prevents ISO C++ const string assignment warnings for pyext sources. "-Wno-writable-strings", + "-Wno-write-strings", ], }) @@ -934,9 +934,41 @@ OBJC_SRCS = [ objc_library( name = "objectivec", hdrs = OBJC_HDRS + OBJC_PRIVATE_HDRS, + copts = [ + "-Wno-vla", + ], includes = [ "objectivec", ], non_arc_srcs = OBJC_SRCS, visibility = ["//visibility:public"], ) + +################################################################################ +# Test generated proto support +################################################################################ + +genrule( + name = "generated_protos", + srcs = ["src/google/protobuf/unittest_import.proto"], + outs = ["unittest_gen.proto"], + cmd = "cat $(SRCS) | sed 's|google/|src/google/|' > $(OUTS)" +) + +proto_library( + name = "generated_protos_proto", + srcs = [ + "unittest_gen.proto", + "src/google/protobuf/unittest_import_public.proto", + ], +) + +py_proto_library( + name = "generated_protos_py", + srcs = [ + "unittest_gen.proto", + "src/google/protobuf/unittest_import_public.proto", + ], + default_runtime = "", + protoc = ":protoc", +) diff --git a/CHANGES.txt b/CHANGES.txt index c7c42916..c18c6377 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,91 @@ +2018-06-01 version 3.6.0 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) + + C++ + * Starting from this release, we now require C++11. For those we cannot yet + upgrade to C++11, we will try to keep the 3.5.x branch updated with + critical bug fixes only. If you have any concerns about this, please + comment on issue #2780. + * Moved to C++11 types like std::atomic and std::unique_ptr and away from our + old custom-built equivalents. + * Added support for repeated message fields in lite protos using implicit + weak fields. This is an experimental feature that allows the linker to + strip out more unused messages than previously was possible. + * Fixed SourceCodeInfo for interpreted options and extension range options. + * Fixed always_print_enums_as_ints option for JSON serialization. + * Added support for ignoring unknown enum values when parsing JSON. + * Create std::string in Arena memory. + * Fixed ValidateDateTime to correctly check the day. + * Fixed bug in ZeroCopyStreamByteSink. + * Various other cleanups and fixes. + + Java + * Dropped support for Java 6. + * Added a UTF-8 decoder that uses Unsafe to directly decode a byte buffer. + * Added deprecation annotations to generated code for deprecated oneof + fields. + * Fixed map field serialization in DynamicMessage. + * Cleanup and documentation for Java Lite runtime. + * Various other fixes and cleanups + * Fixed unboxed arraylists to handle an edge case + * Improved performance for copying between unboxed arraylists + * Fixed lite protobuf to avoid Java compiler warnings + * Improved test coverage for lite runtime + * Performance improvements for lite runtime + + Python + * Fixed bytes/string map key incompatibility between C++ and pure-Python + implementations (issue #4029) + * Added __init__.py files to compiler and util subpackages + * Use /MT for all Windows versions + * Fixed an issue affecting the Python-C++ implementation when used with + Cython (issue #2896) + * Various text format fixes + * Various fixes to resolve behavior differences between the pure-Python and + Python-C++ implementations + + PHP + * Added php_metadata_namespace to control the file path of generated metadata + file. + * Changed generated classes of nested message/enum. E.g., Foo.Bar, which + previously generates Foo_Bar, now generates Foo/Bar + * Added array constructor. When creating a message, users can pass a php + array whose content is field name to value pairs into constructor. The + created message will be initialized according to the array. Note that + message field should use a message value instead of a sub-array. + * Various bug fixes. + + Objective-C + * We removed some helper class methods from GPBDictionary to shrink the size + of the library, the functionary is still there, but you may need to do some + specific +alloc / -init… methods instead. + * Minor improvements in the performance of object field getters/setters by + avoiding some memory management overhead. + * Fix a memory leak during the raising of some errors. + * Make header importing completely order independent. + * Small code improvements for things the undefined behaviors compiler option + was flagging. + + Ruby + * Added ruby_package file option to control the module of generated class. + * Various bug fixes. + + Javascript + * Allow setting string to int64 field. + + Csharp + * Unknown fields are now parsed and then sent back on the wire. They can be + discarded at parse time via a CodedInputStream option. + * Movement towards working with .NET 3.5 and Unity + * Expression trees are no longer used + * AOT generics issues in Unity/il2cpp have a workaround (see this commit for + details) + * Floating point values are now compared bitwise (affects NaN value + comparisons) + * The default size limit when parsing is now 2GB rather than 64MB + * MessageParser now supports parsing from a slice of a byte array + * JSON list parsing now accepts null values where the underlying proto + representation does + 2017-12-20 version 3.5.1 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) Planned Future Changes * Make C++ implementation C++11 only: we plan to require C++11 to build diff --git a/Makefile.am b/Makefile.am index 76bb2ba2..24f520c6 100644 --- a/Makefile.am +++ b/Makefile.am @@ -935,6 +935,7 @@ js_EXTRA_DIST= \ js/commonjs/import_test.js \ js/commonjs/jasmine.json \ js/commonjs/rewrite_tests_for_commonjs.js \ + js/commonjs/strict_test.js \ js/commonjs/test6/test6.proto \ js/commonjs/test7/test7.proto \ js/compatibility_tests/v3.0.0/binary/arith_test.js \ @@ -1006,6 +1007,8 @@ js_EXTRA_DIST= \ js/test4.proto \ js/test5.proto \ js/test8.proto \ + js/test9.proto \ + js/test10.proto \ js/test_bootstrap.js \ js/testbinary.proto \ js/testempty.proto diff --git a/Protobuf.podspec b/Protobuf.podspec index 24498a27..b5bb673e 100644 --- a/Protobuf.podspec +++ b/Protobuf.podspec @@ -5,7 +5,7 @@ # dependent projects use the :git notation to refer to the library. Pod::Spec.new do |s| s.name = 'Protobuf' - s.version = '3.5.2' + s.version = '3.6.0' s.summary = 'Protocol Buffers v.3 runtime library for Objective-C.' s.homepage = 'https://github.com/google/protobuf' s.license = '3-Clause BSD License' diff --git a/appveyor.yml b/appveyor.yml index 1dd076a4..91f737a8 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -17,6 +17,12 @@ environment: UNICODE: ON - platform: Win64 + language: cpp + image: Visual Studio 2017 + BUILD_DLL: OFF + UNICODE: ON + + - platform: Win64 language: csharp image: Visual Studio 2017 @@ -28,6 +28,11 @@ fi set -ex +# The absence of a m4 directory in googletest causes autoreconf to fail when +# building under the CentOS docker image. It's a warning in regular build on +# Ubuntu/gLinux as well. +mkdir -p third_party/googletest/m4 + # TODO(kenton): Remove the ",no-obsolete" part and fix the resulting warnings. autoreconf -f -i -Wall,no-obsolete diff --git a/benchmarks/Makefile.am b/benchmarks/Makefile.am index c7e6ee20..4bf57026 100644 --- a/benchmarks/Makefile.am +++ b/benchmarks/Makefile.am @@ -109,6 +109,7 @@ cpp_benchmark_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir)/cpp -I$(top_srcdir)/thi # so a direct "make test_cpp" could fail if parallel enough. # See: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html#Recording-Dependencies-manually cpp/cpp_benchmark-cpp_benchmark.$(OBJEXT): $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/third_party/benchmark/src/libbenchmark.a +cpp/benchmark-cpp_benchmark.$(OBJEXT): $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/third_party/benchmark/src/libbenchmark.a nodist_cpp_benchmark_SOURCES = \ $(benchmarks_protoc_outputs) \ $(benchmarks_protoc_outputs_proto2) \ diff --git a/cmake/protobuf-config.cmake.in b/cmake/protobuf-config.cmake.in index 18054687..69d63e3e 100644 --- a/cmake/protobuf-config.cmake.in +++ b/cmake/protobuf-config.cmake.in @@ -105,7 +105,7 @@ function(protobuf_generate) add_custom_command( OUTPUT ${_generated_srcs} COMMAND protobuf::protoc - ARGS --${protobuf_generate_LANGUAGE}_out ${_dll_export_decl}${protobuf_generate_PROTOC_OUT_DIR} ${_protobuf_include_path} ${_abs_file} + ARGS --${protobuf_generate_LANGUAGE}_out ${_dll_export_decl}${protobuf_generate_PROTOC_OUT_DIR}/${_rel_dir} ${_protobuf_include_path} ${_abs_file} DEPENDS ${_abs_file} protobuf::protoc COMMENT "Running ${protobuf_generate_LANGUAGE} protocol buffer compiler on ${_proto}" VERBATIM ) diff --git a/configure.ac b/configure.ac index 634c0a69..69b3fe30 100644 --- a/configure.ac +++ b/configure.ac @@ -17,7 +17,7 @@ AC_PREREQ(2.59) # In the SVN trunk, the version should always be the next anticipated release # version with the "-pre" suffix. (We used to use "-SNAPSHOT" but this pushed # the size of one file name in the dist tarfile over the 99-char limit.) -AC_INIT([Protocol Buffers],[3.5.2],[protobuf@googlegroups.com],[protobuf]) +AC_INIT([Protocol Buffers],[3.6.0],[protobuf@googlegroups.com],[protobuf]) AM_MAINTAINER_MODE([enable]) diff --git a/conformance/Makefile.am b/conformance/Makefile.am index 765f3588..e51ab80a 100644 --- a/conformance/Makefile.am +++ b/conformance/Makefile.am @@ -279,8 +279,6 @@ $(protoc_outputs): protoc_middleman $(other_language_protoc_outputs): protoc_middleman -BUILT_SOURCES = $(protoc_outputs) $(other_language_protoc_outputs) - CLEANFILES = $(protoc_outputs) protoc_middleman javac_middleman conformance-java javac_middleman_lite conformance-java-lite conformance-csharp conformance-php conformance-php-c $(other_language_protoc_outputs) MAINTAINERCLEANFILES = \ diff --git a/conformance/conformance.proto b/conformance/conformance.proto index 525140e9..897e7b38 100644 --- a/conformance/conformance.proto +++ b/conformance/conformance.proto @@ -82,6 +82,8 @@ message ConformanceRequest { // protobuf_test_messages.proto3.TestAllTypesProto3 or // protobuf_test_messages.proto2.TestAllTypesProto2. string message_type = 4; + + bool ignore_unknown_json = 5; } // Represents a single test case's output. diff --git a/conformance/conformance_php.php b/conformance/conformance_php.php index 19f9a092..65483e27 100755 --- a/conformance/conformance_php.php +++ b/conformance/conformance_php.php @@ -6,8 +6,8 @@ require_once("Conformance/ConformanceRequest.php"); require_once("Protobuf_test_messages/Proto3/ForeignMessage.php"); require_once("Protobuf_test_messages/Proto3/ForeignEnum.php"); require_once("Protobuf_test_messages/Proto3/TestAllTypesProto3.php"); -require_once("Protobuf_test_messages/Proto3/TestAllTypesProto3_NestedMessage.php"); -require_once("Protobuf_test_messages/Proto3/TestAllTypesProto3_NestedEnum.php"); +require_once("Protobuf_test_messages/Proto3/TestAllTypesProto3/NestedMessage.php"); +require_once("Protobuf_test_messages/Proto3/TestAllTypesProto3/NestedEnum.php"); require_once("GPBMetadata/Conformance.php"); require_once("GPBMetadata/Google/Protobuf/TestMessagesProto3.php"); @@ -39,8 +39,10 @@ function doTest($request) trigger_error("Protobuf request doesn't have specific payload type", E_USER_ERROR); } } elseif ($request->getPayload() == "json_payload") { + $ignore_json_unknown = $request->getIgnoreUnknownJson(); try { - $test_message->mergeFromJsonString($request->getJsonPayload()); + $test_message->mergeFromJsonString($request->getJsonPayload(), + $ignore_json_unknown); } catch (Exception $e) { $response->setParseError($e->getMessage()); return $response; diff --git a/conformance/conformance_test.cc b/conformance/conformance_test.cc index 22bbbfb3..d9e88ce5 100644 --- a/conformance/conformance_test.cc +++ b/conformance/conformance_test.cc @@ -191,6 +191,78 @@ string UpperCase(string str) { namespace google { namespace protobuf { +ConformanceTestSuite::ConformanceRequestSetting::ConformanceRequestSetting( + ConformanceLevel level, conformance::WireFormat input_format, + conformance::WireFormat output_format, bool is_proto3, + const string& test_name, const string& input) + : level_(level), input_format_(input_format), + output_format_(output_format), is_proto3_(is_proto3) { + auto newTestMessage = [&is_proto3]() { + Message* newMessage; + if (is_proto3) { + newMessage = new TestAllTypesProto3; + } else { + newMessage = new TestAllTypesProto2; + } + return newMessage; + }; + + string input_format_string; + string output_format_string; + string rname = is_proto3 ? ".Proto3" : ".Proto2"; + + switch (input_format) { + case conformance::PROTOBUF: { + request_.set_protobuf_payload(input); + input_format_string = ".ProtobufInput."; + break; + } + + case conformance::JSON: { + request_.set_json_payload(input); + input_format_string = ".JsonInput."; + break; + } + + default: + GOOGLE_LOG(FATAL) << "Unspecified input format"; + } + + switch (output_format) { + case conformance::PROTOBUF: { + output_format_string = ".ProtobufOutput"; + break; + } + + case conformance::JSON: { + output_format_string = ".JsonOutput"; + break; + } + + default: + GOOGLE_LOG(FATAL) << "Unspecified output format"; + } + + test_name_ = ConformanceLevelToString(level) + rname + + input_format_string + test_name + + output_format_string; + + if (is_proto3) { + request_.set_message_type("protobuf_test_messages.proto3.TestAllTypesProto3"); + } else { + request_.set_message_type("protobuf_test_messages.proto2.TestAllTypesProto2"); + } + request_.set_requested_output_format(output_format); +} + +Message* ConformanceTestSuite::ConformanceRequestSetting::GetTestMessage() const { + if (is_proto3_) { + return new TestAllTypesProto3(); + } else { + return new TestAllTypesProto2(); + } +} + void ConformanceTestSuite::ReportSuccess(const string& test_name) { if (expected_to_fail_.erase(test_name) != 0) { StringAppendF(&output_, @@ -273,75 +345,36 @@ void ConformanceTestSuite::RunTest(const string& test_name, } void ConformanceTestSuite::RunValidInputTest( - const string& test_name, ConformanceLevel level, const string& input, - WireFormat input_format, const string& equivalent_text_format, - WireFormat requested_output, bool isProto3) { - auto newTestMessage = [&isProto3]() { - Message* newMessage; - if (isProto3) { - newMessage = new TestAllTypesProto3; - } else { - newMessage = new TestAllTypesProto2; - } - return newMessage; - }; - Message* reference_message = newTestMessage(); + const ConformanceRequestSetting& setting, + const string& equivalent_text_format) { + Message* reference_message = setting.GetTestMessage(); GOOGLE_CHECK( TextFormat::ParseFromString(equivalent_text_format, reference_message)) - << "Failed to parse data for test case: " << test_name + << "Failed to parse data for test case: " << setting.GetTestName() << ", data: " << equivalent_text_format; const string equivalent_wire_format = reference_message->SerializeAsString(); - RunValidBinaryInputTest(test_name, level, input, input_format, - equivalent_wire_format, requested_output, isProto3); + RunValidBinaryInputTest(setting, equivalent_wire_format); } void ConformanceTestSuite::RunValidBinaryInputTest( - const string& test_name, ConformanceLevel level, const string& input, - WireFormat input_format, const string& equivalent_wire_format, - WireFormat requested_output, bool isProto3) { - auto newTestMessage = [&isProto3]() { - Message* newMessage; - if (isProto3) { - newMessage = new TestAllTypesProto3; - } else { - newMessage = new TestAllTypesProto2; - } - return newMessage; - }; - Message* reference_message = newTestMessage(); + const ConformanceRequestSetting& setting, + const string& equivalent_wire_format) { + const string& test_name = setting.GetTestName(); + ConformanceLevel level = setting.GetLevel(); + + Message* reference_message = setting.GetTestMessage(); GOOGLE_CHECK( reference_message->ParseFromString(equivalent_wire_format)) << "Failed to parse wire data for test case: " << test_name; - ConformanceRequest request; + const ConformanceRequest& request = setting.GetRequest(); ConformanceResponse response; - switch (input_format) { - case conformance::PROTOBUF: { - request.set_protobuf_payload(input); - if (isProto3) { - request.set_message_type("protobuf_test_messages.proto3.TestAllTypesProto3"); - } else { - request.set_message_type("protobuf_test_messages.proto2.TestAllTypesProto2"); - } - break; - } - - case conformance::JSON: { - request.set_message_type("protobuf_test_messages.proto3.TestAllTypesProto3"); - request.set_json_payload(input); - break; - } - - default: - GOOGLE_LOG(FATAL) << "Unspecified input format"; - } - - request.set_requested_output_format(requested_output); - RunTest(test_name, request, &response); - Message *test_message = newTestMessage(); + Message* test_message = setting.GetTestMessage(); + + WireFormat requested_output = request.requested_output_format(); switch (response.result_case()) { case ConformanceResponse::RESULT_NOT_SET: @@ -476,56 +509,60 @@ void ConformanceTestSuite::ExpectHardParseFailureForProto( void ConformanceTestSuite::RunValidJsonTest( const string& test_name, ConformanceLevel level, const string& input_json, const string& equivalent_text_format) { - RunValidInputTest( - ConformanceLevelToString(level) + ".Proto3.JsonInput." + test_name + - ".ProtobufOutput", level, input_json, conformance::JSON, - equivalent_text_format, conformance::PROTOBUF, true); - RunValidInputTest( - ConformanceLevelToString(level) + ".Proto3.JsonInput." + test_name + - ".JsonOutput", level, input_json, conformance::JSON, - equivalent_text_format, conformance::JSON, true); + ConformanceRequestSetting setting1( + level, conformance::JSON, conformance::PROTOBUF, + true, test_name, input_json); + RunValidInputTest(setting1, equivalent_text_format); + + ConformanceRequestSetting setting2( + level, conformance::JSON, conformance::JSON, + true, test_name, input_json); + RunValidInputTest(setting2, equivalent_text_format); } void ConformanceTestSuite::RunValidJsonTestWithProtobufInput( const string& test_name, ConformanceLevel level, const TestAllTypesProto3& input, const string& equivalent_text_format) { - RunValidInputTest( - ConformanceLevelToString(level) + ".Proto3" + ".ProtobufInput." + test_name + - ".JsonOutput", level, input.SerializeAsString(), conformance::PROTOBUF, - equivalent_text_format, conformance::JSON, true); + ConformanceRequestSetting setting( + level, conformance::PROTOBUF, conformance::JSON, + true, test_name, input.SerializeAsString()); + RunValidInputTest(setting, equivalent_text_format); +} + +void ConformanceTestSuite::RunValidJsonIgnoreUnknownTest( + const string& test_name, ConformanceLevel level, const string& input_json, + const string& equivalent_text_format) { + ConformanceRequestSetting setting( + level, conformance::JSON, conformance::PROTOBUF, + true, test_name, input_json); + setting.SetIgnoreUnknownJson(true); + RunValidInputTest(setting, equivalent_text_format); } void ConformanceTestSuite::RunValidProtobufTest( const string& test_name, ConformanceLevel level, const string& input_protobuf, const string& equivalent_text_format, bool isProto3) { - string rname = ".Proto3"; - if (!isProto3) { - rname = ".Proto2"; - } - RunValidInputTest( - ConformanceLevelToString(level) + rname + ".ProtobufInput." + test_name + - ".ProtobufOutput", level, input_protobuf, conformance::PROTOBUF, - equivalent_text_format, conformance::PROTOBUF, isProto3); + ConformanceRequestSetting setting1( + level, conformance::PROTOBUF, conformance::PROTOBUF, + isProto3, test_name, input_protobuf); + RunValidInputTest(setting1, equivalent_text_format); + if (isProto3) { - RunValidInputTest( - ConformanceLevelToString(level) + rname + ".ProtobufInput." + test_name + - ".JsonOutput", level, input_protobuf, conformance::PROTOBUF, - equivalent_text_format, conformance::JSON, isProto3); + ConformanceRequestSetting setting2( + level, conformance::PROTOBUF, conformance::JSON, + true, test_name, input_protobuf); + RunValidInputTest(setting2, equivalent_text_format); } } void ConformanceTestSuite::RunValidBinaryProtobufTest( const string& test_name, ConformanceLevel level, const string& input_protobuf, bool isProto3) { - string rname = ".Proto3"; - if (!isProto3) { - rname = ".Proto2"; - } - RunValidBinaryInputTest( - ConformanceLevelToString(level) + rname + ".ProtobufInput." + test_name + - ".ProtobufOutput", level, input_protobuf, conformance::PROTOBUF, - input_protobuf, conformance::PROTOBUF, isProto3); + ConformanceRequestSetting setting( + level, conformance::PROTOBUF, conformance::PROTOBUF, + isProto3, test_name, input_protobuf); + RunValidBinaryInputTest(setting, input_protobuf); } void ConformanceTestSuite::RunValidProtobufTestWithMessage( @@ -2535,6 +2572,43 @@ bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner, } )"); + RunValidJsonIgnoreUnknownTest( + "IgnoreUnknownJsonNumber", REQUIRED, + R"({ + "unknown": 1 + })", + ""); + RunValidJsonIgnoreUnknownTest( + "IgnoreUnknownJsonString", REQUIRED, + R"({ + "unknown": "a" + })", + ""); + RunValidJsonIgnoreUnknownTest( + "IgnoreUnknownJsonTrue", REQUIRED, + R"({ + "unknown": true + })", + ""); + RunValidJsonIgnoreUnknownTest( + "IgnoreUnknownJsonFalse", REQUIRED, + R"({ + "unknown": false + })", + ""); + RunValidJsonIgnoreUnknownTest( + "IgnoreUnknownJsonNull", REQUIRED, + R"({ + "unknown": null + })", + ""); + RunValidJsonIgnoreUnknownTest( + "IgnoreUnknownJsonObject", REQUIRED, + R"({ + "unknown": {"a": 1} + })", + ""); + bool ok = true; if (!CheckSetEmpty(expected_to_fail_, "nonexistent_tests.txt", "These tests were listed in the failure list, but they " diff --git a/conformance/conformance_test.h b/conformance/conformance_test.h index 2649f8b2..685f67fb 100644 --- a/conformance/conformance_test.h +++ b/conformance/conformance_test.h @@ -44,6 +44,7 @@ #include <google/protobuf/util/type_resolver.h> #include <google/protobuf/wire_format_lite.h> +#include "conformance.pb.h" #include "third_party/jsoncpp/json.h" namespace conformance { @@ -146,7 +147,42 @@ class ConformanceTestSuite { REQUIRED = 0, RECOMMENDED = 1, }; - string ConformanceLevelToString(ConformanceLevel level); + + class ConformanceRequestSetting { + public: + ConformanceRequestSetting( + ConformanceLevel level, conformance::WireFormat input_format, + conformance::WireFormat output_format, bool is_proto3, + const string& test_name, const string& input); + + Message* GetTestMessage() const; + + const string& GetTestName() const { + return test_name_; + } + + const conformance::ConformanceRequest& GetRequest() const { + return request_; + } + + const ConformanceLevel GetLevel() const { + return level_; + } + + void SetIgnoreUnknownJson(bool ignore_unknown_json) { + request_.set_ignore_unknown_json(ignore_unknown_json); + } + + private: + ConformanceLevel level_; + conformance::WireFormat input_format_; + conformance::WireFormat output_format_; + bool is_proto3_; + string test_name_; + conformance::ConformanceRequest request_; + }; + + static string ConformanceLevelToString(ConformanceLevel level); void ReportSuccess(const std::string& test_name); void ReportFailure(const string& test_name, @@ -160,24 +196,17 @@ class ConformanceTestSuite { void RunTest(const std::string& test_name, const conformance::ConformanceRequest& request, conformance::ConformanceResponse* response); - void RunValidInputTest(const string& test_name, - ConformanceLevel level, - const string& input, - conformance::WireFormat input_format, - const string& equivalent_text_format, - conformance::WireFormat requested_output, - bool isProto3); - void RunValidBinaryInputTest(const string& test_name, - ConformanceLevel level, - const string& input, - conformance::WireFormat input_format, - const string& equivalent_wire_format, - conformance::WireFormat requested_output, - bool isProto3); + void RunValidInputTest(const ConformanceRequestSetting& setting, + const string& equivalent_text_format); + void RunValidBinaryInputTest(const ConformanceRequestSetting& setting, + const string& equivalent_wire_format); void RunValidJsonTest(const string& test_name, ConformanceLevel level, const string& input_json, const string& equivalent_text_format); + void RunValidJsonIgnoreUnknownTest( + const string& test_name, ConformanceLevel level, const string& input_json, + const string& equivalent_text_format); void RunValidJsonTestWithProtobufInput( const string& test_name, ConformanceLevel level, diff --git a/conformance/failure_list_cpp.txt b/conformance/failure_list_cpp.txt index 752fbb5d..ea8e8473 100644 --- a/conformance/failure_list_cpp.txt +++ b/conformance/failure_list_cpp.txt @@ -54,3 +54,9 @@ Required.Proto2.ProtobufInput.PrematureEofInPackedField.SINT32 Required.Proto2.ProtobufInput.PrematureEofInPackedField.SINT64 Required.Proto2.ProtobufInput.PrematureEofInPackedField.UINT32 Required.Proto2.ProtobufInput.PrematureEofInPackedField.UINT64 +Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput diff --git a/conformance/failure_list_csharp.txt b/conformance/failure_list_csharp.txt index 2a20aa78..69c723b5 100644 --- a/conformance/failure_list_csharp.txt +++ b/conformance/failure_list_csharp.txt @@ -1,2 +1,8 @@ Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput diff --git a/conformance/failure_list_java.txt b/conformance/failure_list_java.txt index dc1f9ba5..6f085c66 100644 --- a/conformance/failure_list_java.txt +++ b/conformance/failure_list_java.txt @@ -45,3 +45,9 @@ Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValu Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE +Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput diff --git a/conformance/failure_list_php_c.txt b/conformance/failure_list_php_c.txt index 088708e9..6ecd94fd 100644 --- a/conformance/failure_list_php_c.txt +++ b/conformance/failure_list_php_c.txt @@ -1,8 +1,6 @@ Recommended.FieldMaskNumbersDontRoundTrip.JsonOutput Recommended.FieldMaskPathsDontRoundTrip.JsonOutput Recommended.FieldMaskTooManyUnderscore.JsonOutput -Recommended.Proto3.JsonInput.BoolFieldIntegerOne -Recommended.Proto3.JsonInput.BoolFieldIntegerZero Recommended.Proto3.JsonInput.DurationHas3FractionalDigits.Validator Recommended.Proto3.JsonInput.DurationHas6FractionalDigits.Validator Recommended.Proto3.JsonInput.DurationHas9FractionalDigits.Validator @@ -10,9 +8,6 @@ Recommended.Proto3.JsonInput.DurationHasZeroFractionalDigit.Validator Recommended.Proto3.JsonInput.Int64FieldBeString.Validator Recommended.Proto3.JsonInput.MapFieldValueIsNull Recommended.Proto3.JsonInput.OneofZeroBytes.JsonOutput -Recommended.Proto3.JsonInput.OneofZeroBytes.ProtobufOutput -Recommended.Proto3.JsonInput.OneofZeroString.JsonOutput -Recommended.Proto3.JsonInput.OneofZeroString.ProtobufOutput Recommended.Proto3.JsonInput.RepeatedFieldMessageElementIsNull Recommended.Proto3.JsonInput.RepeatedFieldPrimitiveElementIsNull Recommended.Proto3.JsonInput.StringEndsWithEscapeChar @@ -21,9 +16,6 @@ Recommended.Proto3.JsonInput.StringFieldUnpairedHighSurrogate Recommended.Proto3.JsonInput.StringFieldUnpairedLowSurrogate Recommended.Proto3.JsonInput.Uint64FieldBeString.Validator Recommended.Proto3.ProtobufInput.OneofZeroBytes.JsonOutput -Recommended.Proto3.ProtobufInput.OneofZeroBytes.ProtobufOutput -Recommended.Proto3.ProtobufInput.OneofZeroString.JsonOutput -Recommended.Proto3.ProtobufInput.OneofZeroString.ProtobufOutput Required.DurationProtoInputTooLarge.JsonOutput Required.DurationProtoInputTooSmall.JsonOutput Required.Proto3.JsonInput.Any.JsonOutput @@ -47,63 +39,22 @@ Required.Proto3.JsonInput.AnyWithValueForInteger.ProtobufOutput Required.Proto3.JsonInput.AnyWithValueForJsonObject.JsonOutput Required.Proto3.JsonInput.AnyWithValueForJsonObject.ProtobufOutput Required.Proto3.JsonInput.BoolMapField.JsonOutput -Required.Proto3.JsonInput.DoubleFieldInfinity.JsonOutput -Required.Proto3.JsonInput.DoubleFieldInfinity.ProtobufOutput Required.Proto3.JsonInput.DoubleFieldMaxNegativeValue.JsonOutput Required.Proto3.JsonInput.DoubleFieldMaxNegativeValue.ProtobufOutput -Required.Proto3.JsonInput.DoubleFieldMaxPositiveValue.JsonOutput -Required.Proto3.JsonInput.DoubleFieldMaxPositiveValue.ProtobufOutput -Required.Proto3.JsonInput.DoubleFieldMinNegativeValue.JsonOutput -Required.Proto3.JsonInput.DoubleFieldMinNegativeValue.ProtobufOutput Required.Proto3.JsonInput.DoubleFieldMinPositiveValue.JsonOutput Required.Proto3.JsonInput.DoubleFieldMinPositiveValue.ProtobufOutput Required.Proto3.JsonInput.DoubleFieldNan.JsonOutput -Required.Proto3.JsonInput.DoubleFieldNan.ProtobufOutput -Required.Proto3.JsonInput.DoubleFieldNegativeInfinity.JsonOutput -Required.Proto3.JsonInput.DoubleFieldNegativeInfinity.ProtobufOutput -Required.Proto3.JsonInput.DoubleFieldQuotedValue.JsonOutput -Required.Proto3.JsonInput.DoubleFieldQuotedValue.ProtobufOutput Required.Proto3.JsonInput.DurationMaxValue.JsonOutput Required.Proto3.JsonInput.DurationMaxValue.ProtobufOutput Required.Proto3.JsonInput.DurationMinValue.JsonOutput Required.Proto3.JsonInput.DurationMinValue.ProtobufOutput Required.Proto3.JsonInput.DurationRepeatedValue.JsonOutput Required.Proto3.JsonInput.DurationRepeatedValue.ProtobufOutput -Required.Proto3.JsonInput.EnumFieldNumericValueNonZero.JsonOutput -Required.Proto3.JsonInput.EnumFieldNumericValueNonZero.ProtobufOutput -Required.Proto3.JsonInput.EnumFieldNumericValueZero.JsonOutput -Required.Proto3.JsonInput.EnumFieldNumericValueZero.ProtobufOutput -Required.Proto3.JsonInput.EnumFieldUnknownValue.Validator Required.Proto3.JsonInput.FieldMask.JsonOutput Required.Proto3.JsonInput.FieldMask.ProtobufOutput Required.Proto3.JsonInput.FloatFieldInfinity.JsonOutput -Required.Proto3.JsonInput.FloatFieldInfinity.ProtobufOutput Required.Proto3.JsonInput.FloatFieldNan.JsonOutput -Required.Proto3.JsonInput.FloatFieldNan.ProtobufOutput Required.Proto3.JsonInput.FloatFieldNegativeInfinity.JsonOutput -Required.Proto3.JsonInput.FloatFieldNegativeInfinity.ProtobufOutput -Required.Proto3.JsonInput.FloatFieldQuotedValue.JsonOutput -Required.Proto3.JsonInput.FloatFieldQuotedValue.ProtobufOutput -Required.Proto3.JsonInput.FloatFieldTooLarge -Required.Proto3.JsonInput.FloatFieldTooSmall -Required.Proto3.JsonInput.Int32FieldExponentialFormat.JsonOutput -Required.Proto3.JsonInput.Int32FieldExponentialFormat.ProtobufOutput -Required.Proto3.JsonInput.Int32FieldFloatTrailingZero.JsonOutput -Required.Proto3.JsonInput.Int32FieldFloatTrailingZero.ProtobufOutput -Required.Proto3.JsonInput.Int32FieldMaxFloatValue.JsonOutput -Required.Proto3.JsonInput.Int32FieldMaxFloatValue.ProtobufOutput -Required.Proto3.JsonInput.Int32FieldMinFloatValue.JsonOutput -Required.Proto3.JsonInput.Int32FieldMinFloatValue.ProtobufOutput -Required.Proto3.JsonInput.Int32FieldStringValue.JsonOutput -Required.Proto3.JsonInput.Int32FieldStringValue.ProtobufOutput -Required.Proto3.JsonInput.Int32FieldStringValueEscaped.JsonOutput -Required.Proto3.JsonInput.Int32FieldStringValueEscaped.ProtobufOutput -Required.Proto3.JsonInput.Int64FieldMaxValue.JsonOutput -Required.Proto3.JsonInput.Int64FieldMaxValue.ProtobufOutput -Required.Proto3.JsonInput.Int64FieldMinValue.JsonOutput -Required.Proto3.JsonInput.Int64FieldMinValue.ProtobufOutput -Required.Proto3.JsonInput.MessageField.JsonOutput -Required.Proto3.JsonInput.MessageField.ProtobufOutput Required.Proto3.JsonInput.OptionalBoolWrapper.JsonOutput Required.Proto3.JsonInput.OptionalBoolWrapper.ProtobufOutput Required.Proto3.JsonInput.OptionalBytesWrapper.JsonOutput @@ -130,8 +81,6 @@ Required.Proto3.JsonInput.RepeatedBytesWrapper.JsonOutput Required.Proto3.JsonInput.RepeatedBytesWrapper.ProtobufOutput Required.Proto3.JsonInput.RepeatedDoubleWrapper.JsonOutput Required.Proto3.JsonInput.RepeatedDoubleWrapper.ProtobufOutput -Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingMessagesGotInt -Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt Required.Proto3.JsonInput.RepeatedFloatWrapper.JsonOutput Required.Proto3.JsonInput.RepeatedFloatWrapper.ProtobufOutput Required.Proto3.JsonInput.RepeatedInt32Wrapper.JsonOutput @@ -146,7 +95,6 @@ Required.Proto3.JsonInput.RepeatedUint64Wrapper.JsonOutput Required.Proto3.JsonInput.RepeatedUint64Wrapper.ProtobufOutput Required.Proto3.JsonInput.StringFieldEscape.JsonOutput Required.Proto3.JsonInput.StringFieldEscape.ProtobufOutput -Required.Proto3.JsonInput.StringFieldNotAString Required.Proto3.JsonInput.StringFieldSurrogatePair.JsonOutput Required.Proto3.JsonInput.StringFieldSurrogatePair.ProtobufOutput Required.Proto3.JsonInput.StringFieldUnicodeEscape.JsonOutput @@ -155,10 +103,6 @@ Required.Proto3.JsonInput.StringFieldUnicodeEscapeWithLowercaseHexLetters.JsonOu Required.Proto3.JsonInput.StringFieldUnicodeEscapeWithLowercaseHexLetters.ProtobufOutput Required.Proto3.JsonInput.Struct.JsonOutput Required.Proto3.JsonInput.Struct.ProtobufOutput -Required.Proto3.JsonInput.Uint32FieldMaxFloatValue.JsonOutput -Required.Proto3.JsonInput.Uint32FieldMaxFloatValue.ProtobufOutput -Required.Proto3.JsonInput.Uint64FieldMaxValue.JsonOutput -Required.Proto3.JsonInput.Uint64FieldMaxValue.ProtobufOutput Required.Proto3.JsonInput.ValueAcceptBool.JsonOutput Required.Proto3.JsonInput.ValueAcceptBool.ProtobufOutput Required.Proto3.JsonInput.ValueAcceptFloat.JsonOutput diff --git a/conformance/failure_list_python-post26.txt b/conformance/failure_list_python-post26.txt index 19d99b04..60b1146e 100644 --- a/conformance/failure_list_python-post26.txt +++ b/conformance/failure_list_python-post26.txt @@ -1,2 +1,8 @@ JsonInput.StringFieldSurrogateInWrongOrder JsonInput.StringFieldUnpairedHighSurrogate +Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput diff --git a/conformance/failure_list_python.txt b/conformance/failure_list_python.txt index e3ce7af7..5f01b53b 100644 --- a/conformance/failure_list_python.txt +++ b/conformance/failure_list_python.txt @@ -19,3 +19,9 @@ Required.Proto3.ProtobufInput.IllegalZeroFieldNum_Case_0 Required.Proto3.ProtobufInput.IllegalZeroFieldNum_Case_1 Required.Proto3.ProtobufInput.IllegalZeroFieldNum_Case_2 Required.Proto3.ProtobufInput.IllegalZeroFieldNum_Case_3 +Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput diff --git a/conformance/failure_list_python_cpp.txt b/conformance/failure_list_python_cpp.txt index a498ad1a..f80517d9 100644 --- a/conformance/failure_list_python_cpp.txt +++ b/conformance/failure_list_python_cpp.txt @@ -52,3 +52,9 @@ Required.Proto2.ProtobufInput.PrematureEofInPackedField.SINT32 Required.Proto2.ProtobufInput.PrematureEofInPackedField.SINT64 Required.Proto2.ProtobufInput.PrematureEofInPackedField.UINT32 Required.Proto2.ProtobufInput.PrematureEofInPackedField.UINT64 +Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput diff --git a/conformance/failure_list_ruby.txt b/conformance/failure_list_ruby.txt index b2683372..2d09acd4 100644 --- a/conformance/failure_list_ruby.txt +++ b/conformance/failure_list_ruby.txt @@ -135,3 +135,9 @@ Required.Proto3.ProtobufInput.FloatFieldNormalizeSignalingNan.JsonOutput Required.Proto3.ProtobufInput.ValidDataRepeated.FLOAT.JsonOutput Required.TimestampProtoInputTooLarge.JsonOutput Required.TimestampProtoInputTooSmall.JsonOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput +Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput diff --git a/csharp/Google.Protobuf.Tools.nuspec b/csharp/Google.Protobuf.Tools.nuspec index e3386047..0ba7cc9f 100644 --- a/csharp/Google.Protobuf.Tools.nuspec +++ b/csharp/Google.Protobuf.Tools.nuspec @@ -5,7 +5,7 @@ <title>Google Protocol Buffers tools</title> <summary>Tools for Protocol Buffers - Google's data interchange format.</summary> <description>See project site for more info.</description> - <version>3.5.2</version> + <version>3.6.0</version> <authors>Google Inc.</authors> <owners>protobuf-packages</owners> <licenseUrl>https://github.com/google/protobuf/blob/master/LICENSE</licenseUrl> diff --git a/csharp/src/Google.Protobuf.Conformance/Conformance.cs b/csharp/src/Google.Protobuf.Conformance/Conformance.cs index f6118ea2..77284824 100644 --- a/csharp/src/Google.Protobuf.Conformance/Conformance.cs +++ b/csharp/src/Google.Protobuf.Conformance/Conformance.cs @@ -24,21 +24,22 @@ namespace Conformance { static ConformanceReflection() { byte[] descriptorData = global::System.Convert.FromBase64String( string.Concat( - "ChFjb25mb3JtYW5jZS5wcm90bxILY29uZm9ybWFuY2UiowEKEkNvbmZvcm1h", + "ChFjb25mb3JtYW5jZS5wcm90bxILY29uZm9ybWFuY2UiwAEKEkNvbmZvcm1h", "bmNlUmVxdWVzdBIaChBwcm90b2J1Zl9wYXlsb2FkGAEgASgMSAASFgoManNv", "bl9wYXlsb2FkGAIgASgJSAASOAoXcmVxdWVzdGVkX291dHB1dF9mb3JtYXQY", "AyABKA4yFy5jb25mb3JtYW5jZS5XaXJlRm9ybWF0EhQKDG1lc3NhZ2VfdHlw", - "ZRgEIAEoCUIJCgdwYXlsb2FkIrEBChNDb25mb3JtYW5jZVJlc3BvbnNlEhUK", - "C3BhcnNlX2Vycm9yGAEgASgJSAASGQoPc2VyaWFsaXplX2Vycm9yGAYgASgJ", - "SAASFwoNcnVudGltZV9lcnJvchgCIAEoCUgAEhoKEHByb3RvYnVmX3BheWxv", - "YWQYAyABKAxIABIWCgxqc29uX3BheWxvYWQYBCABKAlIABIRCgdza2lwcGVk", - "GAUgASgJSABCCAoGcmVzdWx0KjUKCldpcmVGb3JtYXQSDwoLVU5TUEVDSUZJ", - "RUQQABIMCghQUk9UT0JVRhABEggKBEpTT04QAkIhCh9jb20uZ29vZ2xlLnBy", - "b3RvYnVmLmNvbmZvcm1hbmNlYgZwcm90bzM=")); + "ZRgEIAEoCRIbChNpZ25vcmVfdW5rbm93bl9qc29uGAUgASgIQgkKB3BheWxv", + "YWQisQEKE0NvbmZvcm1hbmNlUmVzcG9uc2USFQoLcGFyc2VfZXJyb3IYASAB", + "KAlIABIZCg9zZXJpYWxpemVfZXJyb3IYBiABKAlIABIXCg1ydW50aW1lX2Vy", + "cm9yGAIgASgJSAASGgoQcHJvdG9idWZfcGF5bG9hZBgDIAEoDEgAEhYKDGpz", + "b25fcGF5bG9hZBgEIAEoCUgAEhEKB3NraXBwZWQYBSABKAlIAEIICgZyZXN1", + "bHQqNQoKV2lyZUZvcm1hdBIPCgtVTlNQRUNJRklFRBAAEgwKCFBST1RPQlVG", + "EAESCAoESlNPThACQiEKH2NvbS5nb29nbGUucHJvdG9idWYuY29uZm9ybWFu", + "Y2ViBnByb3RvMw==")); descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, new pbr::FileDescriptor[] { }, new pbr::GeneratedClrTypeInfo(new[] {typeof(global::Conformance.WireFormat), }, new pbr::GeneratedClrTypeInfo[] { - new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.ConformanceRequest), global::Conformance.ConformanceRequest.Parser, new[]{ "ProtobufPayload", "JsonPayload", "RequestedOutputFormat", "MessageType" }, new[]{ "Payload" }, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.ConformanceRequest), global::Conformance.ConformanceRequest.Parser, new[]{ "ProtobufPayload", "JsonPayload", "RequestedOutputFormat", "MessageType", "IgnoreUnknownJson" }, new[]{ "Payload" }, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.ConformanceResponse), global::Conformance.ConformanceResponse.Parser, new[]{ "ParseError", "SerializeError", "RuntimeError", "ProtobufPayload", "JsonPayload", "Skipped" }, new[]{ "Result" }, null, null) })); } @@ -89,6 +90,7 @@ namespace Conformance { public ConformanceRequest(ConformanceRequest other) : this() { requestedOutputFormat_ = other.requestedOutputFormat_; messageType_ = other.messageType_; + ignoreUnknownJson_ = other.ignoreUnknownJson_; switch (other.PayloadCase) { case PayloadOneofCase.ProtobufPayload: ProtobufPayload = other.ProtobufPayload; @@ -158,6 +160,17 @@ namespace Conformance { } } + /// <summary>Field number for the "ignore_unknown_json" field.</summary> + public const int IgnoreUnknownJsonFieldNumber = 5; + private bool ignoreUnknownJson_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public bool IgnoreUnknownJson { + get { return ignoreUnknownJson_; } + set { + ignoreUnknownJson_ = value; + } + } + private object payload_; /// <summary>Enum of possible cases for the "payload" oneof.</summary> public enum PayloadOneofCase { @@ -194,6 +207,7 @@ namespace Conformance { if (JsonPayload != other.JsonPayload) return false; if (RequestedOutputFormat != other.RequestedOutputFormat) return false; if (MessageType != other.MessageType) return false; + if (IgnoreUnknownJson != other.IgnoreUnknownJson) return false; if (PayloadCase != other.PayloadCase) return false; return Equals(_unknownFields, other._unknownFields); } @@ -205,6 +219,7 @@ namespace Conformance { if (payloadCase_ == PayloadOneofCase.JsonPayload) hash ^= JsonPayload.GetHashCode(); if (RequestedOutputFormat != 0) hash ^= RequestedOutputFormat.GetHashCode(); if (MessageType.Length != 0) hash ^= MessageType.GetHashCode(); + if (IgnoreUnknownJson != false) hash ^= IgnoreUnknownJson.GetHashCode(); hash ^= (int) payloadCase_; if (_unknownFields != null) { hash ^= _unknownFields.GetHashCode(); @@ -235,6 +250,10 @@ namespace Conformance { output.WriteRawTag(34); output.WriteString(MessageType); } + if (IgnoreUnknownJson != false) { + output.WriteRawTag(40); + output.WriteBool(IgnoreUnknownJson); + } if (_unknownFields != null) { _unknownFields.WriteTo(output); } @@ -255,6 +274,9 @@ namespace Conformance { if (MessageType.Length != 0) { size += 1 + pb::CodedOutputStream.ComputeStringSize(MessageType); } + if (IgnoreUnknownJson != false) { + size += 1 + 1; + } if (_unknownFields != null) { size += _unknownFields.CalculateSize(); } @@ -272,6 +294,9 @@ namespace Conformance { if (other.MessageType.Length != 0) { MessageType = other.MessageType; } + if (other.IgnoreUnknownJson != false) { + IgnoreUnknownJson = other.IgnoreUnknownJson; + } switch (other.PayloadCase) { case PayloadOneofCase.ProtobufPayload: ProtobufPayload = other.ProtobufPayload; @@ -308,6 +333,10 @@ namespace Conformance { MessageType = input.ReadString(); break; } + case 40: { + IgnoreUnknownJson = input.ReadBool(); + break; + } } } } diff --git a/csharp/src/Google.Protobuf.Test/WellKnownTypes/TimestampTest.cs b/csharp/src/Google.Protobuf.Test/WellKnownTypes/TimestampTest.cs index 9ecd24c6..b8c07ef5 100644 --- a/csharp/src/Google.Protobuf.Test/WellKnownTypes/TimestampTest.cs +++ b/csharp/src/Google.Protobuf.Test/WellKnownTypes/TimestampTest.cs @@ -111,5 +111,106 @@ namespace Google.Protobuf.WellKnownTypes var duration = new Timestamp { Seconds = 1, Nanos = -1 }; Assert.AreEqual("{ \"@warning\": \"Invalid Timestamp\", \"seconds\": \"1\", \"nanos\": -1 }", duration.ToString()); } + + [Test] + public void Comparability() + { + Timestamp + a = null, + b = new Timestamp { Seconds = 1, Nanos = 1 }, + c = new Timestamp { Seconds = 1, Nanos = 10 }, + d = new Timestamp { Seconds = 10, Nanos = 1 }, + e = new Timestamp { Seconds = 10, Nanos = 10 }; + + Assert.IsTrue(b.CompareTo(a) > 0); // null is always first (according to default behavior of Array.Sort) + Assert.IsTrue(b.CompareTo(b) == 0); + Assert.IsTrue(b.CompareTo(b.Clone()) == 0); + Assert.IsTrue(b.CompareTo(c) < 0); + Assert.IsTrue(b.CompareTo(d) < 0); + Assert.IsTrue(b.CompareTo(e) < 0); + + Assert.IsTrue(c.CompareTo(a) > 0); + Assert.IsTrue(c.CompareTo(b) > 0); + Assert.IsTrue(c.CompareTo(c) == 0); + Assert.IsTrue(c.CompareTo(c.Clone()) == 0); + Assert.IsTrue(c.CompareTo(d) < 0); + Assert.IsTrue(c.CompareTo(e) < 0); + + Assert.IsTrue(d.CompareTo(a) > 0); + Assert.IsTrue(d.CompareTo(b) > 0); + Assert.IsTrue(d.CompareTo(c) > 0); + Assert.IsTrue(d.CompareTo(d) == 0); + Assert.IsTrue(d.CompareTo(d.Clone()) == 0); + Assert.IsTrue(d.CompareTo(e) < 0); + + Assert.IsTrue(e.CompareTo(a) > 0); + Assert.IsTrue(e.CompareTo(b) > 0); + Assert.IsTrue(e.CompareTo(c) > 0); + Assert.IsTrue(e.CompareTo(d) > 0); + Assert.IsTrue(e.CompareTo(e) == 0); + Assert.IsTrue(e.CompareTo(e.Clone()) == 0); + } + + + [Test] + public void ComparabilityOperators() + { + Timestamp + a = null, + b = new Timestamp { Seconds = 1, Nanos = 1 }, + c = new Timestamp { Seconds = 1, Nanos = 10 }, + d = new Timestamp { Seconds = 10, Nanos = 1 }, + e = new Timestamp { Seconds = 10, Nanos = 10 }; + +#pragma warning disable CS1718 // Comparison made to same variable + Assert.IsTrue(b > a); + Assert.IsTrue(b == b); + Assert.IsTrue(b == b.Clone()); + Assert.IsTrue(b < c); + Assert.IsTrue(b < d); + Assert.IsTrue(b < e); + + Assert.IsTrue(c > a); + Assert.IsTrue(c > b); + Assert.IsTrue(c == c); + Assert.IsTrue(c == c.Clone()); + Assert.IsTrue(c < d); + Assert.IsTrue(c < e); + + Assert.IsTrue(d > a); + Assert.IsTrue(d > b); + Assert.IsTrue(d > c); + Assert.IsTrue(d == d); + Assert.IsTrue(d == d.Clone()); + Assert.IsTrue(d < e); + + Assert.IsTrue(e > a); + Assert.IsTrue(e > b); + Assert.IsTrue(e > c); + Assert.IsTrue(e > d); + Assert.IsTrue(e == e); + Assert.IsTrue(e == e.Clone()); + + Assert.IsTrue(b >= a); + Assert.IsTrue(b <= c); + Assert.IsTrue(b <= d); + Assert.IsTrue(b <= e); + + Assert.IsTrue(c >= a); + Assert.IsTrue(c >= b); + Assert.IsTrue(c <= d); + Assert.IsTrue(c <= e); + + Assert.IsTrue(d >= a); + Assert.IsTrue(d >= b); + Assert.IsTrue(d >= c); + Assert.IsTrue(d <= e); + + Assert.IsTrue(e >= a); + Assert.IsTrue(e >= b); + Assert.IsTrue(e >= c); + Assert.IsTrue(e >= d); +#pragma warning restore CS1718 // Comparison made to same variable + } } } diff --git a/csharp/src/Google.Protobuf/Google.Protobuf.csproj b/csharp/src/Google.Protobuf/Google.Protobuf.csproj index 93dcd854..46728b72 100644 --- a/csharp/src/Google.Protobuf/Google.Protobuf.csproj +++ b/csharp/src/Google.Protobuf/Google.Protobuf.csproj @@ -4,7 +4,7 @@ <Description>C# runtime library for Protocol Buffers - Google's data interchange format.</Description> <Copyright>Copyright 2015, Google Inc.</Copyright> <AssemblyTitle>Google Protocol Buffers</AssemblyTitle> - <VersionPrefix>3.5.2</VersionPrefix> + <VersionPrefix>3.6.0</VersionPrefix> <Authors>Google Inc.</Authors> <TargetFrameworks>netstandard1.0;net45</TargetFrameworks> <GenerateDocumentationFile>true</GenerateDocumentationFile> diff --git a/csharp/src/Google.Protobuf/Reflection/ReflectionUtil.cs b/csharp/src/Google.Protobuf/Reflection/ReflectionUtil.cs index 18a70b80..feaeba0e 100644 --- a/csharp/src/Google.Protobuf/Reflection/ReflectionUtil.cs +++ b/csharp/src/Google.Protobuf/Reflection/ReflectionUtil.cs @@ -34,10 +34,6 @@ using Google.Protobuf.Compatibility; using System; using System.Reflection; -#if NET35 -using Google.Protobuf.Compatibility; -#endif - namespace Google.Protobuf.Reflection { /// <summary> diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/TimestampPartial.cs b/csharp/src/Google.Protobuf/WellKnownTypes/TimestampPartial.cs index aa403473..a9251974 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/TimestampPartial.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/TimestampPartial.cs @@ -36,7 +36,7 @@ using System.Text; namespace Google.Protobuf.WellKnownTypes { - public partial class Timestamp : ICustomDiagnosticMessage + public partial class Timestamp : ICustomDiagnosticMessage, IComparable<Timestamp> { private static readonly DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); // Constants determined programmatically, but then hard-coded so they can be constant expressions. @@ -223,6 +223,109 @@ namespace Google.Protobuf.WellKnownTypes } /// <summary> + /// Given another timestamp, returns 0 if the timestamps are equivalent, -1 if this timestamp precedes the other, and 1 otherwise + /// </summary> + /// <remarks> + /// Make sure the timestamps are normalized. Comparing non-normalized timestamps is not specified and may give unexpected results. + /// </remarks> + /// <param name="other">Timestamp to compare</param> + /// <returns>an integer indicating whether this timestamp precedes or follows the other</returns> + public int CompareTo(Timestamp other) + { + return other == null ? 1 + : Seconds < other.Seconds ? -1 + : Seconds > other.Seconds ? 1 + : Nanos < other.Nanos ? -1 + : Nanos > other.Nanos ? 1 + : 0; + } + + /// <summary> + /// Compares two timestamps and returns whether the first is less than (chronologically precedes) the second + /// </summary> + /// <remarks> + /// Make sure the timestamps are normalized. Comparing non-normalized timestamps is not specified and may give unexpected results. + /// </remarks> + /// <param name="a"></param> + /// <param name="b"></param> + /// <returns>true if a precedes b</returns> + public static bool operator <(Timestamp a, Timestamp b) + { + return a.CompareTo(b) < 0; + } + + /// <summary> + /// Compares two timestamps and returns whether the first is greater than (chronologically follows) the second + /// </summary> + /// <remarks> + /// Make sure the timestamps are normalized. Comparing non-normalized timestamps is not specified and may give unexpected results. + /// </remarks> + /// <param name="a"></param> + /// <param name="b"></param> + /// <returns>true if a follows b</returns> + public static bool operator >(Timestamp a, Timestamp b) + { + return a.CompareTo(b) > 0; + } + + /// <summary> + /// Compares two timestamps and returns whether the first is less than (chronologically precedes) the second + /// </summary> + /// <remarks> + /// Make sure the timestamps are normalized. Comparing non-normalized timestamps is not specified and may give unexpected results. + /// </remarks> + /// <param name="a"></param> + /// <param name="b"></param> + /// <returns>true if a precedes b</returns> + public static bool operator <=(Timestamp a, Timestamp b) + { + return a.CompareTo(b) <= 0; + } + + /// <summary> + /// Compares two timestamps and returns whether the first is greater than (chronologically follows) the second + /// </summary> + /// <remarks> + /// Make sure the timestamps are normalized. Comparing non-normalized timestamps is not specified and may give unexpected results. + /// </remarks> + /// <param name="a"></param> + /// <param name="b"></param> + /// <returns>true if a follows b</returns> + public static bool operator >=(Timestamp a, Timestamp b) + { + return a.CompareTo(b) >= 0; + } + + + /// <summary> + /// Returns whether two timestamps are equivalent + /// </summary> + /// <remarks> + /// Make sure the timestamps are normalized. Comparing non-normalized timestamps is not specified and may give unexpected results. + /// </remarks> + /// <param name="a"></param> + /// <param name="b"></param> + /// <returns>true if the two timestamps refer to the same nanosecond</returns> + public static bool operator ==(Timestamp a, Timestamp b) + { + return ReferenceEquals(a, b) || (a is null ? (b is null ? true : false) : a.Equals(b)); + } + + /// <summary> + /// Returns whether two timestamps differ + /// </summary> + /// <remarks> + /// Make sure the timestamps are normalized. Comparing non-normalized timestamps is not specified and may give unexpected results. + /// </remarks> + /// <param name="a"></param> + /// <param name="b"></param> + /// <returns>true if the two timestamps differ</returns> + public static bool operator !=(Timestamp a, Timestamp b) + { + return !(a == b); + } + + /// <summary> /// Returns a string representation of this <see cref="Timestamp"/> for diagnostic purposes. /// </summary> /// <remarks> diff --git a/docs/options.md b/docs/options.md new file mode 100644 index 00000000..c9b2290a --- /dev/null +++ b/docs/options.md @@ -0,0 +1,186 @@ +# Protobuf Global Extension Registry + +This file contains a global registry of known extensions for descriptor.proto, +so that any developer who wishes to use multiple 3rd party projects, each with +their own extensions, can be confident that there won't be collisions in +extension numbers. + +If you need an extension number for your custom option (see [custom options]( +https://developers.google.com/protocol-buffers/docs/proto#customoptions)), +please [send us a pull request](https://github.com/google/protobuf/pulls) to +add an entry to this doc, or [create an issue](https://github.com/google/protobuf/issues) +with info about your project (name and website) so we can add an entry for you. + +## Existing Registered Extensions + +1. C# port of protocol buffers + * Website: http://github.com/jskeet/dotnet-protobufs + * Extensions: 1000 + +1. Perl/XS port of protocol buffers + * Website: http://code.google.com/p/protobuf-perlxs + * Extensions: 1001 + +1. Objective-C port of protocol buffers + * Website: http://code.google.com/p/protobuf-objc + * Extensions: 1002 + +1. Google Wave Federation Protocol open-source release (FedOne) + * Website: http://code.google.com/p/wave-protocol + * Extensions: 1003 + +1. PHP code generator plugin + * Website: ??? + * Extensions: 1004 + +1. GWT code generator plugin (third-party!) + * Website: http://code.google.com/p/protobuf-gwt/ + * Extensions: 1005 + +1. Unix Domain RPC code generator plugin + * Website: http://go/udrpc + * Extensions: 1006 + +1. Object-C generator plugin (Plausible Labs) + * Website: http://www.plausible.coop + * Extensions: 1007 + +1. TBD (code42.com) + * Website: ??? + * Extensions: 1008 + +1. Goby Underwater Autonomy Project + * Website: https://launchpad.net/goby + * Extensions: 1009 + +1. Nanopb + * Website: http://kapsi.fi/~jpa/nanopb + * Extensions: 1010 + +1. Bluefin AUV Communication Extensions + * Website: http://www.bluefinrobotics.com + * Extensions: 1011 + +1. Dynamic Compact Control Language + * Website: https://launchpad.net/dccl + * Extensions: 1012 + +1. ScaleOut StateServer® Native C++ API + * Website: http://www.scaleoutsoftware.com + * Extensions: 1013 + +1. FoundationDB SQL Layer + * Website: https://github.com/FoundationDB/sql-layer + * Extensions: 1014 + +1. Fender + * Website: https://github.com/hassox/fender + * Extensions: 1015 + +1. Vortex + * Website: http://www.prismtech.com/vortex + * Extensions: 1016 + +1. tresorit + * Website: https://tresorit.com/ + * Extensions: 1017 + +1. CRIU (Checkpoint Restore In Userspace) + * Website: http://criu.org/Main_Page + * Extensions: 1018 + +1. protobuf-c + * Website: https://github.com/protobuf-c/protobuf-c + * Extensions: 1019 + +1. ScalePB + * Website: http://trueaccord.github.io/ScalaPB/ + * Extensions: 1020 + +1. protoc-gen-bq-schema + * Website: https://github.com/GoogleCloudPlatform/protoc-gen-bq-schema + * Extensions: 1021 + +1. grpc-gateway + * Website: https://github.com/gengo/grpc-gateway + * Extensions: 1022 + +1. Certificate Transparency + * Website: https://github.com/google/certificate-transparency + * Extensions: 1023 + +1. JUNOS Router Telemetry + * Website: http://www.juniper.net + * Extensions: 1024 + +1. Spine Event Engine + * Website: https://github.com/SpineEventEngine/core-java + * Extensions: 1025 + +1. Aruba cloud platform + * Website: ??? + * Extensions: 1026 -> 1030 + +1. Voltha + * Website: ??? + * Extensions: 1031 -> 1033 + +1. gator + * Website: ??? + * Extensions: 1034 + +1. protoc-gen-flowtypes + * Website: https://github.com/tmc/grpcutil/tree/master/protoc-gen-flowtypes + * Extensions: 1035 + +1. ProfaneDB + * Website: https://gitlab.com/ProfaneDB/ProfaneDB + * Extensions: 1036 + +1. protobuf-net + * Website: https://github.com/mgravell/protobuf-net + * Extensions: 1037 + +1. FICO / StreamEngine + * Website: http://www.fico.com/ + * Extensions: 1038 + +1. GopherJS + * Website: https://github.com/johanbrandhorst/protobuf + * Extensions: 1039 + +1. ygot + * Website: https://github.com/openconfig/ygot + * Extensions: 1040 + +1. go-grpcmw + * Website: https://github.com/MarquisIO/go-grpcmw + * Extensions: 1041 + +1. grpc-gateway protoc-gen-swagger + * Website: https://github.com/grpc-ecosystem/grpc-gateway + * Extensions: 1042 + +1. AN Message + * Website: TBD + * Extensions: 1043 + +1. protofire + * Website: https://github.com/ribrdb/protofire + * Extensions: 1044 + +1. Gravity + * Website: https://github.com/aphysci/gravity + * Extensions: 1045 + +1. SEMI Standards – I&C Technical Committee + * Website: http://downloads.semi.org/web/wstdsbal.nsf/9c2b317e76523cca88257641005a47f5/88a5863a580e323088256e7b00707489!OpenDocument + * Extensions: 1046 + +1. Elixir plugin + * Website: https://github.com/tony612/grpc-elixir + * Extensions: 1047 + +1. API client generators + * Website: ??? + * Extensions: 1048-1056 diff --git a/docs/third_party.md b/docs/third_party.md index 4eeb2787..7b0fb2df 100644 --- a/docs/third_party.md +++ b/docs/third_party.md @@ -23,6 +23,7 @@ These are projects we know about implementing Protocol Buffers for other program * C#: https://silentorbit.com/protobuf/ * C#/.NET/WCF/VB: http://code.google.com/p/protobuf-net/ * Clojure: http://github.com/ninjudd/clojure-protobuf +* Clojure: https://github.com/clojusc/protobuf * Common Lisp: http://github.com/ndantam/s-protobuf * Common Lisp: http://github.com/brown/protobuf * D: https://github.com/dcarp/protobuf-d @@ -127,6 +128,7 @@ GRPC (http://www.grpc.io/) is Google's RPC implementation for Protocol Buffers. * https://github.com/johanbrandhorst/protobuf (GopherJS) * https://github.com/awakesecurity/gRPC-haskell (Haskell) * https://github.com/Yeolar/raster (C++) +* https://github.com/jnordberg/wsrpc (JavaScript Node.js/Browser) ## Other Utilities diff --git a/examples/WORKSPACE b/examples/WORKSPACE index 936f2441..153a5932 100644 --- a/examples/WORKSPACE +++ b/examples/WORKSPACE @@ -5,8 +5,6 @@ http_archive( strip_prefix = "protobuf-master", urls = ["https://github.com/google/protobuf/archive/master.zip"], ) -load("@com_google_protobuf//:protobuf.bzl", "check_protobuf_required_bazel_version") -check_protobuf_required_bazel_version() # This com_google_protobuf_cc repository is required for cc_proto_library # rule. It provides protobuf C++ runtime. Note that it actually is the same @@ -33,3 +31,13 @@ http_archive( strip_prefix = "protobuf-javalite", urls = ["https://github.com/google/protobuf/archive/javalite.zip"], ) + +http_archive( + name = "bazel_skylib", + sha256 = "bbccf674aa441c266df9894182d80de104cabd19be98be002f6d478aaa31574d", + strip_prefix = "bazel-skylib-2169ae1c374aab4a09aa90e65efe1a3aad4e279b", + urls = ["https://github.com/bazelbuild/bazel-skylib/archive/2169ae1c374aab4a09aa90e65efe1a3aad4e279b.tar.gz"], +) + +load("@bazel_skylib//:lib.bzl", "versions") +versions.check(minimum_bazel_version = "0.5.4") diff --git a/examples/list_people_test.go b/examples/list_people_test.go index 87d6ad6e..887997fe 100644 --- a/examples/list_people_test.go +++ b/examples/list_people_test.go @@ -34,7 +34,7 @@ func TestWritePersonWritesPerson(t *testing.T) { func TestListPeopleWritesList(t *testing.T) { buf := new(bytes.Buffer) - in := pb.AddressBook{[]*pb.Person{ + in := pb.AddressBook{People: []*pb.Person { { Name: "John Doe", Id: 101, diff --git a/java/core/pom.xml b/java/core/pom.xml index 5b0b9520..a7d4ea37 100644 --- a/java/core/pom.xml +++ b/java/core/pom.xml @@ -6,7 +6,7 @@ <parent> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-parent</artifactId> - <version>3.5.2</version> + <version>3.6.0</version> </parent> <artifactId>protobuf-java</artifactId> diff --git a/java/pom.xml b/java/pom.xml index f2284918..35d653b4 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -11,7 +11,7 @@ <groupId>com.google.protobuf</groupId> <artifactId>protobuf-parent</artifactId> - <version>3.5.2</version> + <version>3.6.0</version> <packaging>pom</packaging> <name>Protocol Buffers [Parent]</name> diff --git a/java/util/pom.xml b/java/util/pom.xml index 4e55df0d..8ea4a40d 100644 --- a/java/util/pom.xml +++ b/java/util/pom.xml @@ -6,7 +6,7 @@ <parent> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-parent</artifactId> - <version>3.5.2</version> + <version>3.6.0</version> </parent> <artifactId>protobuf-java-util</artifactId> diff --git a/js/README.md b/js/README.md index ef0d4b19..d8edca37 100644 --- a/js/README.md +++ b/js/README.md @@ -144,6 +144,7 @@ Some examples: The `import_style` option is left to the default, which is `closure`. - `--js_out=import_style=commonjs,binary:protos`: this contains the options `import_style=commonjs` and `binary` and outputs to the directory `protos`. + `import_style=commonjs_strict` doesn't expose the output on the global scope. API === diff --git a/js/commonjs/strict_test.js b/js/commonjs/strict_test.js new file mode 100644 index 00000000..46458c10 --- /dev/null +++ b/js/commonjs/strict_test.js @@ -0,0 +1,67 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2016 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Test suite is written using Jasmine -- see http://jasmine.github.io/ + + + +var googleProtobuf = require('google-protobuf'); +var asserts = require('closure_asserts_commonjs'); +var global = Function('return this')(); + +// Bring asserts into the global namespace. +googleProtobuf.object.extend(global, asserts); + +var test9_pb = require('./test9_pb'); +var test10_pb = require('./test10_pb'); + +describe('Strict test suite', function() { + it('testImportedMessage', function() { + var simple1 = new test9_pb.jspb.exttest.strict.nine.Simple9() + var simple2 = new test9_pb.jspb.exttest.strict.nine.Simple9() + assertObjectEquals(simple1.toObject(), simple2.toObject()); + }); + + it('testGlobalScopePollution', function() { + assertObjectEquals(global.jspb.exttest, undefined); + }); + + describe('with imports', function() { + it('testImportedMessage', function() { + var simple1 = new test10_pb.jspb.exttest.strict.ten.Simple10() + var simple2 = new test10_pb.jspb.exttest.strict.ten.Simple10() + assertObjectEquals(simple1.toObject(), simple2.toObject()); + }); + + it('testGlobalScopePollution', function() { + assertObjectEquals(global.jspb.exttest, undefined); + }); + }); +}); diff --git a/js/gulpfile.js b/js/gulpfile.js index fc9559f9..709c5cf9 100644 --- a/js/gulpfile.js +++ b/js/gulpfile.js @@ -41,6 +41,12 @@ var group2Protos = [ 'commonjs/test7/test7.proto', ]; +var group3Protos = [ + 'test9.proto', + 'test10.proto' +]; + + gulp.task('genproto_well_known_types_closure', function (cb) { exec(protoc + ' --js_out=one_output_file_per_input_file,binary:. -I ../src -I . ' + wellKnownTypes.join(' '), function (err, stdout, stderr) { @@ -112,6 +118,15 @@ gulp.task('genproto_wellknowntypes', function (cb) { cb(err); }); }); +gulp.task('genproto_group3_commonjs_strict', function (cb) { + exec('mkdir -p commonjs_out && ' + protoc + ' --js_out=import_style=commonjs_strict,binary:commonjs_out -I ../src -I commonjs -I . ' + group3Protos.join(' '), + function (err, stdout, stderr) { + console.log(stdout); + console.log(stderr); + cb(err); + }); +}); + function getClosureBuilderCommand(exportsFile, outputFile) { return './node_modules/google-closure-library/closure/bin/build/closurebuilder.py ' + @@ -159,7 +174,7 @@ gulp.task('commonjs_testdeps', function (cb) { }); }); -gulp.task('make_commonjs_out', ['dist', 'genproto_well_known_types_commonjs', 'genproto_group1_commonjs', 'genproto_group2_commonjs', 'genproto_commonjs_wellknowntypes', 'commonjs_asserts', 'commonjs_testdeps'], function (cb) { +gulp.task('make_commonjs_out', ['dist', 'genproto_well_known_types_commonjs', 'genproto_group1_commonjs', 'genproto_group2_commonjs', 'genproto_commonjs_wellknowntypes', 'commonjs_asserts', 'commonjs_testdeps', 'genproto_group3_commonjs_strict'], function (cb) { // TODO(haberman): minify this more aggressively. // Will require proper externs/exports. var cmd = "mkdir -p commonjs_out/binary && mkdir -p commonjs_out/test_node_modules && "; @@ -174,6 +189,7 @@ gulp.task('make_commonjs_out', ['dist', 'genproto_well_known_types_commonjs', 'g exec(cmd + 'cp commonjs/jasmine.json commonjs_out/jasmine.json && ' + 'cp google-protobuf.js commonjs_out/test_node_modules && ' + + 'cp commonjs/strict_test.js commonjs_out/strict_test.js &&' + 'cp commonjs/import_test.js commonjs_out/import_test.js', function (err, stdout, stderr) { console.log(stdout); @@ -443,7 +443,8 @@ jspb.Map.prototype.serializeBinary = function( /** * Read one key/value message from the given BinaryReader. Compatible as the * `reader` callback parameter to jspb.BinaryReader.readMessage, to be called - * when a key/value pair submessage is encountered. + * when a key/value pair submessage is encountered. If the Key is undefined, + * we should default it to 0. * @template K, V * @param {!jspb.Map} map * @param {!jspb.BinaryReader} reader @@ -457,12 +458,17 @@ jspb.Map.prototype.serializeBinary = function( * readMessage, in which case the second callback arg form is used. * * @param {?function(V,!jspb.BinaryReader)=} opt_valueReaderCallback - * The BinaryReader parsing callback for type V, if V is a message type. + * The BinaryReader parsing callback for type V, if V is a message type + * + * @param {K=} opt_defaultKey + * The default value for the type of map keys. Accepting map + * entries with unset keys is required for maps to be backwards compatible + * with the repeated message representation described here: goo.gl/zuoLAC * */ jspb.Map.deserializeBinary = function(map, reader, keyReaderFn, valueReaderFn, - opt_valueReaderCallback) { - var key = undefined; + opt_valueReaderCallback, opt_defaultKey) { + var key = opt_defaultKey; var value = undefined; while (reader.nextField()) { @@ -470,6 +476,7 @@ jspb.Map.deserializeBinary = function(map, reader, keyReaderFn, valueReaderFn, break; } var field = reader.getFieldNumber(); + if (field == 1) { // Key. key = keyReaderFn.call(reader); diff --git a/js/maps_test.js b/js/maps_test.js index e8dd2f21..e496f446 100755 --- a/js/maps_test.js +++ b/js/maps_test.js @@ -35,6 +35,11 @@ goog.require('goog.userAgent'); goog.require('proto.jspb.test.MapValueEnum'); goog.require('proto.jspb.test.MapValueMessage'); goog.require('proto.jspb.test.TestMapFields'); +goog.require('proto.jspb.test.TestMapFieldsOptionalKeys'); +goog.require('proto.jspb.test.MapEntryOptionalKeysStringKey'); +goog.require('proto.jspb.test.MapEntryOptionalKeysInt32Key'); +goog.require('proto.jspb.test.MapEntryOptionalKeysInt64Key'); +goog.require('proto.jspb.test.MapEntryOptionalKeysBoolKey'); // CommonJS-LoadFromFile: test_pb proto.jspb.test goog.require('proto.jspb.test.MapValueMessageNoBinary'); @@ -76,7 +81,7 @@ function toArray(iter) { * Helper: generate test methods for this TestMapFields class. * @param {?} msgInfo * @param {?} submessageCtor - * @param {!string} suffix + * @param {string} suffix */ function makeTests(msgInfo, submessageCtor, suffix) { /** @@ -260,6 +265,39 @@ function makeTests(msgInfo, submessageCtor, suffix) { var decoded = msgInfo.deserializeBinary(serialized); checkMapFields(decoded); }); + /** + * Tests deserialization of undefined map keys go to default values in binary format. + */ + it('testMapDeserializationForUndefinedKeys', function() { + var testMessageOptionalKeys = new proto.jspb.test.TestMapFieldsOptionalKeys(); + var mapEntryStringKey = new proto.jspb.test.MapEntryOptionalKeysStringKey(); + mapEntryStringKey.setValue("a"); + testMessageOptionalKeys.setMapStringString(mapEntryStringKey); + var mapEntryInt32Key = new proto.jspb.test.MapEntryOptionalKeysInt32Key(); + mapEntryInt32Key.setValue("b"); + testMessageOptionalKeys.setMapInt32String(mapEntryInt32Key); + var mapEntryInt64Key = new proto.jspb.test.MapEntryOptionalKeysInt64Key(); + mapEntryInt64Key.setValue("c"); + testMessageOptionalKeys.setMapInt64String(mapEntryInt64Key); + var mapEntryBoolKey = new proto.jspb.test.MapEntryOptionalKeysBoolKey(); + mapEntryBoolKey.setValue("d"); + testMessageOptionalKeys.setMapBoolString(mapEntryBoolKey); + var deserializedMessage = msgInfo.deserializeBinary( + testMessageOptionalKeys.serializeBinary() + ); + checkMapEquals(deserializedMessage.getMapStringStringMap(), [ + ['', 'a'] + ]); + checkMapEquals(deserializedMessage.getMapInt32StringMap(), [ + [0, 'b'] + ]); + checkMapEquals(deserializedMessage.getMapInt64StringMap(), [ + [0, 'c'] + ]); + checkMapEquals(deserializedMessage.getMapBoolStringMap(), [ + [false, 'd'] + ]); + }); } diff --git a/js/package.json b/js/package.json index 39d99f93..325f2dcc 100644 --- a/js/package.json +++ b/js/package.json @@ -1,6 +1,6 @@ { "name": "google-protobuf", - "version": "3.5.2", + "version": "3.6.0", "description": "Protocol Buffers for JavaScript", "main": "google-protobuf.js", "files": [ diff --git a/js/test10.proto b/js/test10.proto new file mode 100644 index 00000000..9fa5256c --- /dev/null +++ b/js/test10.proto @@ -0,0 +1,39 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package jspb.exttest.strict.ten; + +import "test9.proto"; + +message Simple10 { + jspb.exttest.strict.nine.Simple9 a = 1; +} diff --git a/js/test9.proto b/js/test9.proto new file mode 100644 index 00000000..9f680852 --- /dev/null +++ b/js/test9.proto @@ -0,0 +1,39 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; + +package jspb.exttest.strict.nine; + +message Simple9 { + required string a_string = 1; + repeated string a_repeated_string = 2; + optional bool a_boolean = 3; +} diff --git a/js/testbinary.proto b/js/testbinary.proto index 116f17fb..ed5bdfc6 100644 --- a/js/testbinary.proto +++ b/js/testbinary.proto @@ -201,6 +201,38 @@ message TestMapFields { map<string, TestMapFields> map_string_testmapfields = 12; } +// These proto are 'mock map' entries to test the above map deserializing with +// undefined keys. Make sure TestMapFieldsOptionalKeys is written to be +// deserialized by TestMapFields +message MapEntryOptionalKeysStringKey { + optional string key = 1; + optional string value = 2; +} + +message MapEntryOptionalKeysInt32Key { + optional int32 key = 1; + optional string value = 2; +} + +message MapEntryOptionalKeysInt64Key { + optional int64 key = 1; + optional string value = 2; +} + +message MapEntryOptionalKeysBoolKey { + optional bool key = 1; + optional string value = 2; +} + +message TestMapFieldsOptionalKeys { + optional MapEntryOptionalKeysStringKey map_string_string = 1; + optional MapEntryOptionalKeysInt32Key map_int32_string= 8; + optional MapEntryOptionalKeysInt64Key map_int64_string = 9; + optional MapEntryOptionalKeysBoolKey map_bool_string = 10; +} + +// End mock-map entries + enum MapValueEnum { MAP_VALUE_FOO = 0; MAP_VALUE_BAR = 1; diff --git a/kokoro/linux/bazel/build.sh b/kokoro/linux/bazel/build.sh index d8aea724..e0c1ee42 100755 --- a/kokoro/linux/bazel/build.sh +++ b/kokoro/linux/bazel/build.sh @@ -7,3 +7,6 @@ cd $(dirname $0)/../../.. git submodule update --init --recursive bazel test :protobuf_test + +cd examples +bazel build :all diff --git a/kokoro/linux/build_and_run_docker.sh b/kokoro/linux/build_and_run_docker.sh index 6a1f327a..f9c60710 100755 --- a/kokoro/linux/build_and_run_docker.sh +++ b/kokoro/linux/build_and_run_docker.sh @@ -35,6 +35,8 @@ echo $git_root docker run \ "$@" \ -e CCACHE_DIR=$CCACHE_DIR \ + -e KOKORO_BUILD_NUMBER=$KOKORO_BUILD_NUMBER \ + -e KOKORO_BUILD_ID=$KOKORO_BUILD_ID \ -e EXTERNAL_GIT_ROOT="/var/local/kokoro/protobuf" \ -e TEST_SET="$TEST_SET" \ -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \ diff --git a/kokoro/linux/cpp_distcheck/build.sh b/kokoro/linux/cpp_distcheck/build.sh index b8b57e35..89b8379e 100755 --- a/kokoro/linux/cpp_distcheck/build.sh +++ b/kokoro/linux/cpp_distcheck/build.sh @@ -5,7 +5,15 @@ # Change to repo root cd $(dirname $0)/../../.. -# Prepare worker environment to run tests -source kokoro/linux/prepare_build_linux_rc - ./tests.sh cpp_distcheck + +# Run tests under release docker image. +DOCKER_IMAGE_NAME=protobuf/protoc_$(sha1sum protoc-artifacts/Dockerfile | cut -f1 -d " ") +docker pull $DOCKER_IMAGE_NAME + +docker run -v $(pwd):/var/local/protobuf --rm $DOCKER_IMAGE_NAME \ + bash -l /var/local/protobuf/tests.sh cpp || FAILED="true" + +if [ "$FAILED" = "true" ]; then + exit 1 +fi diff --git a/kokoro/linux/pull_request_in_docker.sh b/kokoro/linux/pull_request_in_docker.sh index df3636cc..e28ef958 100755 --- a/kokoro/linux/pull_request_in_docker.sh +++ b/kokoro/linux/pull_request_in_docker.sh @@ -5,7 +5,7 @@ WORKSPACE_BASE=`pwd` MY_DIR="$(dirname "$0")" -TEST_SCRIPT=$MY_DIR/../../tests.sh +TEST_SCRIPT=./tests.sh BUILD_DIR=/tmp/protobuf set -e # exit immediately on error @@ -53,7 +53,7 @@ $TIME_CMD $TEST_SCRIPT cpp > >(tee $CPP_STDOUT) 2> >(tee $CPP_STDERR >&2) parallel --results $LOG_OUTPUT_DIR --joblog $OUTPUT_DIR/joblog $TEST_SCRIPT ::: \ $TEST_SET \ - || true # Process test results even if tests fail. + || FAILED="true" # Process test results even if tests fail. cat $OUTPUT_DIR/joblog @@ -67,3 +67,7 @@ TESTOUTPUT_XML_FILE=$COPY_FROM_DOCKER/sponge_log.xml python $MY_DIR/make_test_output.py $OUTPUT_DIR > $TESTOUTPUT_XML_FILE ls -l $TESTOUTPUT_XML_FILE + +if [ "$FAILED" == "true" ]; then + exit 1 +fi diff --git a/kokoro/release/csharp/windows/build_nuget.bat b/kokoro/release/csharp/windows/build_nuget.bat new file mode 100644 index 00000000..bc15bd57 --- /dev/null +++ b/kokoro/release/csharp/windows/build_nuget.bat @@ -0,0 +1,5 @@ +@rem enter repo root +cd /d %~dp0\..\..\..\.. + +cd csharp\src +call build_packages.bat diff --git a/kokoro/release/csharp/windows/release.cfg b/kokoro/release/csharp/windows/release.cfg new file mode 100644 index 00000000..f508c65b --- /dev/null +++ b/kokoro/release/csharp/windows/release.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/release/csharp/windows/build_nuget.bat" +timeout_mins: 60 + +action { + define_artifacts { + regex: "**/*.nupkg" + } +} diff --git a/kokoro/release/protoc/linux/build.sh b/kokoro/release/protoc/linux/build.sh new file mode 100755 index 00000000..6c7049d7 --- /dev/null +++ b/kokoro/release/protoc/linux/build.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -x + +# Change to repo root. +cd $(dirname $0)/../../../.. + +# Initialize any submodules. +git submodule update --init --recursive + +# Generate the configure script. +./autogen.sh + +# Use docker image to build linux artifacts. +DOCKER_IMAGE_NAME=protobuf/protoc_$(sha1sum protoc-artifacts/Dockerfile | cut -f1 -d " ") +docker pull $DOCKER_IMAGE_NAME + +docker run -v $(pwd):/var/local/protobuf --rm $DOCKER_IMAGE_NAME \ + bash -l /var/local/protobuf/protoc-artifacts/build-protoc.sh \ + linux x86_64 protoc || { + echo "Failed to build protoc for linux + x86_64." + exit 1 +} + +docker run -v $(pwd):/var/local/protobuf --rm $DOCKER_IMAGE_NAME \ + bash -l /var/local/protobuf/protoc-artifacts/build-protoc.sh \ + linux x86_32 protoc || { + echo "Failed to build protoc for linux + x86_32." + exit 1 +} + +# Cross-build for some architectures. +# TODO(xiaofeng): It currently fails with "machine `aarch64' not recognized" +# error. +# sudo apt install -y g++-aarch64-linux-gnu +# protoc-artifacts/build-protoc.sh linux aarch_64 protoc diff --git a/kokoro/release/protoc/linux/release.cfg b/kokoro/release/protoc/linux/release.cfg new file mode 100644 index 00000000..4d8b7b61 --- /dev/null +++ b/kokoro/release/protoc/linux/release.cfg @@ -0,0 +1,7 @@ +build_file: "protobuf/kokoro/release/protoc/linux/build.sh" + +action { + define_artifacts { + regex: "**/protoc.exe" + } +} diff --git a/kokoro/release/protoc/macos/build.sh b/kokoro/release/protoc/macos/build.sh new file mode 100644 index 00000000..db9d3140 --- /dev/null +++ b/kokoro/release/protoc/macos/build.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -x +CXXFLAGS_COMMON="-DNDEBUG -mmacosx-version-min=10.9" + +cd github/protobuf +./autogen.sh + +mkdir build32 && cd build32 +export CXXFLAGS="$CXXFLAGS_COMMON -m32" +../configure --disable-shared +make -j4 +file src/protoc +otool -L src/protoc | grep dylib +cd .. + +mkdir build64 && cd build64 +export CXXFLAGS="$CXXFLAGS_COMMON -m64" +../configure --disable-shared +make -j4 +file src/protoc +otool -L src/protoc | grep dylib +cd .. diff --git a/kokoro/release/protoc/macos/release.cfg b/kokoro/release/protoc/macos/release.cfg new file mode 100644 index 00000000..da903d03 --- /dev/null +++ b/kokoro/release/protoc/macos/release.cfg @@ -0,0 +1,8 @@ +# Configuration for Mac OSX protoc release builds +build_file: "protobuf/kokoro/release/protoc/macos/build.sh" + +action { + define_artifacts { + regex: "**/protoc" + } +} diff --git a/kokoro/release/protoc/windows/build.bat b/kokoro/release/protoc/windows/build.bat new file mode 100644 index 00000000..ae0fa94a --- /dev/null +++ b/kokoro/release/protoc/windows/build.bat @@ -0,0 +1,27 @@ +set PATH=C:\Program Files (x86)\MSBuild\14.0\bin\;%PATH% +set generator32=Visual Studio 14 +set generator64=Visual Studio 14 Win64 +set vcplatform32=win32 +set vcplatform64=x64 +set configuration=Release + +echo Building protoc +cd github\protobuf + +mkdir build32 +cd build32 +cmake -G "%generator32%" -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF -Dprotobuf_UNICODE=ON ../cmake +msbuild protobuf.sln /p:Platform=%vcplatform32% || goto error +cd .. + +mkdir build64 +cd build64 +cmake -G "%generator64%" -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF -Dprotobuf_UNICODE=ON ../cmake +msbuild protobuf.sln /p:Platform=%vcplatform64% || goto error +cd .. + +goto :EOF + +:error +echo Failed! +exit /b %ERRORLEVEL% diff --git a/kokoro/release/protoc/windows/release.cfg b/kokoro/release/protoc/windows/release.cfg new file mode 100644 index 00000000..0d0da8f1 --- /dev/null +++ b/kokoro/release/protoc/windows/release.cfg @@ -0,0 +1,8 @@ +# Configuration for Windows protoc release builds +build_file: "protobuf/kokoro/release/protoc/windows/build.bat" + +action { + define_artifacts { + regex: "**/protoc.exe" + } +} diff --git a/kokoro/release/ruby/linux/build_artifacts.sh b/kokoro/release/ruby/linux/build_artifacts.sh new file mode 100755 index 00000000..88555226 --- /dev/null +++ b/kokoro/release/ruby/linux/build_artifacts.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -ex + +# change to repo root +cd $(dirname $0)/../../../.. + +source kokoro/release/ruby/linux/prepare_build.sh + +# ruby environment +source kokoro/release/ruby/linux/ruby/ruby_build_environment.sh + +# build artifacts +bash kokoro/release/ruby/linux/ruby/ruby_build.sh diff --git a/kokoro/release/ruby/linux/prepare_build.sh b/kokoro/release/ruby/linux/prepare_build.sh new file mode 100755 index 00000000..d3ff8764 --- /dev/null +++ b/kokoro/release/ruby/linux/prepare_build.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Move docker's storage location to scratch disk so we don't run out of space. +echo 'DOCKER_OPTS="${DOCKER_OPTS} --graph=/tmpfs/docker"' | sudo tee --append /etc/default/docker +# Use container registry mirror for pulling docker images (should make downloads faster) +# See https://cloud.google.com/container-registry/docs/using-dockerhub-mirroring +echo 'DOCKER_OPTS="${DOCKER_OPTS} --registry-mirror=https://mirror.gcr.io"' | sudo tee --append /etc/default/docker +sudo service docker restart + +# Download Docker images from DockerHub +export DOCKERHUB_ORGANIZATION=grpctesting + +# All artifacts come here +mkdir artifacts +export ARTIFACT_DIR=$(pwd)/artifacts + diff --git a/kokoro/release/ruby/linux/release.cfg b/kokoro/release/ruby/linux/release.cfg new file mode 100644 index 00000000..dbc71b2c --- /dev/null +++ b/kokoro/release/ruby/linux/release.cfg @@ -0,0 +1,8 @@ +# Configuration for Linux release builds +build_file: "protobuf/kokoro/release/ruby/linux/build_artifacts.sh" + +action { + define_artifacts { + regex: "github/protobuf/artifacts/**" + } +} diff --git a/kokoro/release/ruby/linux/ruby/ruby_build.sh b/kokoro/release/ruby/linux/ruby/ruby_build.sh new file mode 100755 index 00000000..cf6b433d --- /dev/null +++ b/kokoro/release/ruby/linux/ruby/ruby_build.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Build protoc +if test ! -e src/protoc; then + ./autogen.sh + ./configure + make -j4 +fi + +umask 0022 +pushd ruby +bundle install && bundle exec rake gem:native +ls pkg +mv pkg/* $ARTIFACT_DIR +popd diff --git a/kokoro/release/ruby/linux/ruby/ruby_build_environment.sh b/kokoro/release/ruby/linux/ruby/ruby_build_environment.sh new file mode 100755 index 00000000..ea04f905 --- /dev/null +++ b/kokoro/release/ruby/linux/ruby/ruby_build_environment.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set +ex +[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh +set -e # rvm commands are very verbose +rvm --default use ruby-2.4.1 +gem install bundler --update +set -ex diff --git a/kokoro/release/ruby/macos/build_artifacts.sh b/kokoro/release/ruby/macos/build_artifacts.sh new file mode 100755 index 00000000..c68b63cc --- /dev/null +++ b/kokoro/release/ruby/macos/build_artifacts.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -ex + +# change to repo root +cd $(dirname $0)/../../../.. + +# all artifacts come here +mkdir artifacts +export ARTIFACT_DIR=$(pwd)/artifacts + +# ruby environment +bash kokoro/release/ruby/macos/ruby/ruby_build_environment.sh + +gem install rubygems-update +update_rubygems + +# build artifacts +bash kokoro/release/ruby/macos/ruby/ruby_build.sh diff --git a/kokoro/release/ruby/macos/release.cfg b/kokoro/release/ruby/macos/release.cfg new file mode 100644 index 00000000..cb1c08bb --- /dev/null +++ b/kokoro/release/ruby/macos/release.cfg @@ -0,0 +1,8 @@ +# Configuration for Mac OSX release builds +build_file: "protobuf/kokoro/release/ruby/macos/build_artifacts.sh" + +action { + define_artifacts { + regex: "github/protobuf/artifacts/**" + } +} diff --git a/kokoro/release/ruby/macos/ruby/ruby_build.sh b/kokoro/release/ruby/macos/ruby/ruby_build.sh new file mode 100755 index 00000000..cf6b433d --- /dev/null +++ b/kokoro/release/ruby/macos/ruby/ruby_build.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Build protoc +if test ! -e src/protoc; then + ./autogen.sh + ./configure + make -j4 +fi + +umask 0022 +pushd ruby +bundle install && bundle exec rake gem:native +ls pkg +mv pkg/* $ARTIFACT_DIR +popd diff --git a/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh b/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh new file mode 100755 index 00000000..e9c619cd --- /dev/null +++ b/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +set -ex + +set +ex # rvm script is very verbose and exits with errorcode +source $HOME/.rvm/scripts/rvm +set -e # rvm commands are very verbose +time rvm install 2.5.0 +rvm use 2.5.0 --default +gem install rake-compiler --no-ri --no-doc +gem install bundler --no-ri --no-doc +rvm osx-ssl-certs status all +rvm osx-ssl-certs update all +set -ex + +rm -rf ~/.rake-compiler + +CROSS_RUBY=$(mktemp tmpfile.XXXXXXXX) + +curl https://raw.githubusercontent.com/rake-compiler/rake-compiler/v1.0.3/tasks/bin/cross-ruby.rake > "$CROSS_RUBY" + +# See https://github.com/grpc/grpc/issues/12161 for verconf.h patch details +patch "$CROSS_RUBY" << EOF +--- cross-ruby.rake 2018-04-10 11:32:16.000000000 -0700 ++++ patched 2018-04-10 11:40:25.000000000 -0700 +@@ -133,8 +133,10 @@ + "--host=#{MINGW_HOST}", + "--target=#{MINGW_TARGET}", + "--build=#{RUBY_BUILD}", +- '--enable-shared', ++ '--enable-static', ++ '--disable-shared', + '--disable-install-doc', ++ '--without-gmp', + '--with-ext=' + ] + +@@ -151,6 +153,7 @@ + # make + file "#{USER_HOME}/builds/#{MINGW_HOST}/#{RUBY_CC_VERSION}/ruby.exe" => ["#{USER_HOME}/builds/#{MINGW_HOST}/#{RUBY_CC_VERSION}/Makefile"] do |t| + chdir File.dirname(t.prerequisites.first) do ++ sh "test -s verconf.h || rm -f verconf.h" # if verconf.h has size 0, make sure it gets re-built by make + sh MAKE + end + end +EOF + +MAKE="make -j8" + +for v in 2.5.0 2.4.0 2.3.0 2.2.2 2.1.6 2.0.0-p645 ; do + ccache -c + rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=x86_64-darwin11 MAKE="$MAKE" +done + +sed 's/x86_64-darwin-11/universal-darwin/' ~/.rake-compiler/config.yml > "$CROSS_RUBY" +mv "$CROSS_RUBY" ~/.rake-compiler/config.yml + diff --git a/objectivec/Tests/GPBTestUtilities.m b/objectivec/Tests/GPBTestUtilities.m index ebccaac9..0362bdde 100644 --- a/objectivec/Tests/GPBTestUtilities.m +++ b/objectivec/Tests/GPBTestUtilities.m @@ -85,7 +85,7 @@ const uint32_t kGPBDefaultRepeatCount = 2; NSLog(@"Wrote data file to %@", path); #else // Kill off the unused variable warning. - dataToWrite = dataToWrite; + (void)dataToWrite; #endif return data; } diff --git a/php/ext/google/protobuf/encode_decode.c b/php/ext/google/protobuf/encode_decode.c index a8c47f4d..31a345b3 100644 --- a/php/ext/google/protobuf/encode_decode.c +++ b/php/ext/google/protobuf/encode_decode.c @@ -1587,8 +1587,11 @@ PHP_METHOD(Message, mergeFromJsonString) { char *data = NULL; PHP_PROTO_SIZE data_len; + zend_bool ignore_json_unknown = false; - if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "s", &data, &data_len) == + if (zend_parse_parameters( + ZEND_NUM_ARGS() TSRMLS_CC, "s|b", &data, &data_len, + &ignore_json_unknown) == FAILURE) { return; } @@ -1607,7 +1610,7 @@ PHP_METHOD(Message, mergeFromJsonString) { stackenv_init(&se, "Error occurred during parsing: %s"); upb_sink_reset(&sink, get_fill_handlers(desc), msg); - parser = upb_json_parser_create(&se.env, method, &sink); + parser = upb_json_parser_create(&se.env, method, &sink, ignore_json_unknown); upb_bufsrc_putbuf(data, data_len, upb_json_parser_input(parser)); stackenv_uninit(&se); diff --git a/php/ext/google/protobuf/message.c b/php/ext/google/protobuf/message.c index e28e42a1..9363191d 100644 --- a/php/ext/google/protobuf/message.c +++ b/php/ext/google/protobuf/message.c @@ -283,6 +283,7 @@ void build_class_from_descriptor( // ----------------------------------------------------------------------------- void Message_construct(zval* msg, zval* array_wrapper) { + TSRMLS_FETCH(); zend_class_entry* ce = Z_OBJCE_P(msg); MessageHeader* intern = NULL; if (EXPECTED(class_added(ce))) { diff --git a/php/ext/google/protobuf/package.xml b/php/ext/google/protobuf/package.xml index 3a556acf..28d713f3 100644 --- a/php/ext/google/protobuf/package.xml +++ b/php/ext/google/protobuf/package.xml @@ -10,11 +10,11 @@ <email>protobuf-opensource@google.com</email> <active>yes</active> </lead> - <date>2018-03-06</date> + <date>2018-06-06</date> <time>11:02:07</time> <version> - <release>3.5.2</release> - <api>3.5.2</api> + <release>3.6.0</release> + <api>3.6.0</api> </version> <stability> <release>stable</release> @@ -232,5 +232,21 @@ GA release. G A release. </notes> </release> + <release> + <version> + <release>3.6.0</release> + <api>3.6.0</api> + </version> + <stability> + <release>stable</release> + <api>stable</api> + </stability> + <date>2018-06-06</date> + <time>11:02:07</time> + <license uri="https://opensource.org/licenses/BSD-3-Clause">3-Clause BSD License</license> + <notes> +G A release. + </notes> + </release> </changelog> </package> diff --git a/php/ext/google/protobuf/protobuf.h b/php/ext/google/protobuf/protobuf.h index c15bde2d..20035ab7 100644 --- a/php/ext/google/protobuf/protobuf.h +++ b/php/ext/google/protobuf/protobuf.h @@ -37,7 +37,7 @@ #include "upb.h" #define PHP_PROTOBUF_EXTNAME "protobuf" -#define PHP_PROTOBUF_VERSION "3.5.2" +#define PHP_PROTOBUF_VERSION "3.6.0" #define MAX_LENGTH_OF_INT64 20 #define SIZEOF_INT64 8 diff --git a/php/ext/google/protobuf/upb.c b/php/ext/google/protobuf/upb.c index e01f3bfd..4644430e 100644 --- a/php/ext/google/protobuf/upb.c +++ b/php/ext/google/protobuf/upb.c @@ -14344,6 +14344,9 @@ struct upb_json_parser { /* Intermediate result of parsing a unicode escape sequence. */ uint32_t digit; + + /* Whether to proceed if unknown field is met. */ + bool ignore_json_unknown; }; struct upb_json_parsermethod { @@ -14864,6 +14867,11 @@ static bool end_number(upb_json_parser *p, const char *ptr) { return false; } + if (p->top->f == NULL) { + multipart_end(p); + return true; + } + return parse_number(p, false); } @@ -15016,6 +15024,10 @@ static bool parse_number(upb_json_parser *p, bool is_quoted) { static bool parser_putbool(upb_json_parser *p, bool val) { bool ok; + if (p->top->f == NULL) { + return true; + } + if (upb_fielddef_type(p->top->f) != UPB_TYPE_BOOL) { upb_status_seterrf(&p->status, "Boolean value specified for non-bool field: %s", @@ -15031,7 +15043,10 @@ static bool parser_putbool(upb_json_parser *p, bool val) { } static bool start_stringval(upb_json_parser *p) { - UPB_ASSERT(p->top->f); + if (p->top->f == NULL) { + multipart_startaccum(p); + return true; + } if (upb_fielddef_isstring(p->top->f)) { upb_jsonparser_frame *inner; @@ -15082,6 +15097,11 @@ static bool start_stringval(upb_json_parser *p) { static bool end_stringval(upb_json_parser *p) { bool ok = true; + if (p->top->f == NULL) { + multipart_end(p); + return true; + } + switch (upb_fielddef_type(p->top->f)) { case UPB_TYPE_BYTES: if (!base64_push(p, getsel_for_handlertype(p, UPB_HANDLER_STRING), @@ -15273,6 +15293,10 @@ static bool handle_mapentry(upb_json_parser *p) { static bool end_membername(upb_json_parser *p) { UPB_ASSERT(!p->top->f); + if (!p->top->m) { + return true; + } + if (p->top->is_map) { return handle_mapentry(p); } else { @@ -15285,9 +15309,10 @@ static bool end_membername(upb_json_parser *p) { multipart_end(p); return true; + } else if (p->ignore_json_unknown) { + multipart_end(p); + return true; } else { - /* TODO(haberman): Ignore unknown fields if requested/configured to do - * so. */ upb_status_seterrf(&p->status, "No such field: %.*s\n", (int)len, buf); upb_env_reporterror(p->env, &p->status); return false; @@ -15319,7 +15344,18 @@ static void end_member(upb_json_parser *p) { } static bool start_subobject(upb_json_parser *p) { - UPB_ASSERT(p->top->f); + if (p->top->f == NULL) { + upb_jsonparser_frame *inner; + if (!check_stack(p)) return false; + + inner = p->top + 1; + inner->m = NULL; + inner->f = NULL; + inner->is_map = false; + inner->is_mapentry = false; + p->top = inner; + return true; + } if (upb_fielddef_ismap(p->top->f)) { upb_jsonparser_frame *inner; @@ -15378,9 +15414,12 @@ static void end_subobject(upb_json_parser *p) { upb_sink_endseq(&p->top->sink, sel); } else { upb_selector_t sel; + bool is_unknown = p->top->m == NULL; p->top--; - sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSUBMSG); - upb_sink_endsubmsg(&p->top->sink, sel); + if (!is_unknown) { + sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSUBMSG); + upb_sink_endsubmsg(&p->top->sink, sel); + } } } @@ -15462,11 +15501,11 @@ static void end_object(upb_json_parser *p) { * final state once, when the closing '"' is seen. */ -#line 1310 "upb/json/parser.rl" +#line 1349 "upb/json/parser.rl" -#line 1222 "upb/json/parser.c" +#line 1261 "upb/json/parser.c" static const char _json_actions[] = { 0, 1, 0, 1, 2, 1, 3, 1, 5, 1, 6, 1, 7, 1, 8, 1, @@ -15615,7 +15654,7 @@ static const int json_en_value_machine = 27; static const int json_en_main = 1; -#line 1313 "upb/json/parser.rl" +#line 1352 "upb/json/parser.rl" size_t parse(void *closure, const void *hd, const char *buf, size_t size, const upb_bufhandle *handle) { @@ -15637,7 +15676,7 @@ size_t parse(void *closure, const void *hd, const char *buf, size_t size, capture_resume(parser, buf); -#line 1393 "upb/json/parser.c" +#line 1432 "upb/json/parser.c" { int _klen; unsigned int _trans; @@ -15712,118 +15751,118 @@ _match: switch ( *_acts++ ) { case 0: -#line 1225 "upb/json/parser.rl" +#line 1264 "upb/json/parser.rl" { p--; {cs = stack[--top]; goto _again;} } break; case 1: -#line 1226 "upb/json/parser.rl" +#line 1265 "upb/json/parser.rl" { p--; {stack[top++] = cs; cs = 10; goto _again;} } break; case 2: -#line 1230 "upb/json/parser.rl" +#line 1269 "upb/json/parser.rl" { start_text(parser, p); } break; case 3: -#line 1231 "upb/json/parser.rl" +#line 1270 "upb/json/parser.rl" { CHECK_RETURN_TOP(end_text(parser, p)); } break; case 4: -#line 1237 "upb/json/parser.rl" +#line 1276 "upb/json/parser.rl" { start_hex(parser); } break; case 5: -#line 1238 "upb/json/parser.rl" +#line 1277 "upb/json/parser.rl" { hexdigit(parser, p); } break; case 6: -#line 1239 "upb/json/parser.rl" +#line 1278 "upb/json/parser.rl" { CHECK_RETURN_TOP(end_hex(parser)); } break; case 7: -#line 1245 "upb/json/parser.rl" +#line 1284 "upb/json/parser.rl" { CHECK_RETURN_TOP(escape(parser, p)); } break; case 8: -#line 1251 "upb/json/parser.rl" +#line 1290 "upb/json/parser.rl" { p--; {cs = stack[--top]; goto _again;} } break; case 9: -#line 1254 "upb/json/parser.rl" +#line 1293 "upb/json/parser.rl" { {stack[top++] = cs; cs = 19; goto _again;} } break; case 10: -#line 1256 "upb/json/parser.rl" +#line 1295 "upb/json/parser.rl" { p--; {stack[top++] = cs; cs = 27; goto _again;} } break; case 11: -#line 1261 "upb/json/parser.rl" +#line 1300 "upb/json/parser.rl" { start_member(parser); } break; case 12: -#line 1262 "upb/json/parser.rl" +#line 1301 "upb/json/parser.rl" { CHECK_RETURN_TOP(end_membername(parser)); } break; case 13: -#line 1265 "upb/json/parser.rl" +#line 1304 "upb/json/parser.rl" { end_member(parser); } break; case 14: -#line 1271 "upb/json/parser.rl" +#line 1310 "upb/json/parser.rl" { start_object(parser); } break; case 15: -#line 1274 "upb/json/parser.rl" +#line 1313 "upb/json/parser.rl" { end_object(parser); } break; case 16: -#line 1280 "upb/json/parser.rl" +#line 1319 "upb/json/parser.rl" { CHECK_RETURN_TOP(start_array(parser)); } break; case 17: -#line 1284 "upb/json/parser.rl" +#line 1323 "upb/json/parser.rl" { end_array(parser); } break; case 18: -#line 1289 "upb/json/parser.rl" +#line 1328 "upb/json/parser.rl" { start_number(parser, p); } break; case 19: -#line 1290 "upb/json/parser.rl" +#line 1329 "upb/json/parser.rl" { CHECK_RETURN_TOP(end_number(parser, p)); } break; case 20: -#line 1292 "upb/json/parser.rl" +#line 1331 "upb/json/parser.rl" { CHECK_RETURN_TOP(start_stringval(parser)); } break; case 21: -#line 1293 "upb/json/parser.rl" +#line 1332 "upb/json/parser.rl" { CHECK_RETURN_TOP(end_stringval(parser)); } break; case 22: -#line 1295 "upb/json/parser.rl" +#line 1334 "upb/json/parser.rl" { CHECK_RETURN_TOP(parser_putbool(parser, true)); } break; case 23: -#line 1297 "upb/json/parser.rl" +#line 1336 "upb/json/parser.rl" { CHECK_RETURN_TOP(parser_putbool(parser, false)); } break; case 24: -#line 1299 "upb/json/parser.rl" +#line 1338 "upb/json/parser.rl" { /* null value */ } break; case 25: -#line 1301 "upb/json/parser.rl" +#line 1340 "upb/json/parser.rl" { CHECK_RETURN_TOP(start_subobject(parser)); } break; case 26: -#line 1302 "upb/json/parser.rl" +#line 1341 "upb/json/parser.rl" { end_subobject(parser); } break; case 27: -#line 1307 "upb/json/parser.rl" +#line 1346 "upb/json/parser.rl" { p--; {cs = stack[--top]; goto _again;} } break; -#line 1579 "upb/json/parser.c" +#line 1618 "upb/json/parser.c" } } @@ -15836,7 +15875,7 @@ _again: _out: {} } -#line 1334 "upb/json/parser.rl" +#line 1373 "upb/json/parser.rl" if (p != pe) { upb_status_seterrf(&parser->status, "Parse error at '%.*s'\n", pe - p, p); @@ -15877,13 +15916,13 @@ static void json_parser_reset(upb_json_parser *p) { /* Emit Ragel initialization of the parser. */ -#line 1633 "upb/json/parser.c" +#line 1672 "upb/json/parser.c" { cs = json_start; top = 0; } -#line 1374 "upb/json/parser.rl" +#line 1413 "upb/json/parser.rl" p->current_state = cs; p->parser_top = top; accumulate_clear(p); @@ -15970,7 +16009,8 @@ static void add_jsonname_table(upb_json_parsermethod *m, const upb_msgdef* md) { upb_json_parser *upb_json_parser_create(upb_env *env, const upb_json_parsermethod *method, - upb_sink *output) { + upb_sink *output, + bool ignore_json_unknown) { #ifndef NDEBUG const size_t size_before = upb_env_bytesallocated(env); #endif @@ -15989,6 +16029,8 @@ upb_json_parser *upb_json_parser_create(upb_env *env, p->top->m = upb_handlers_msgdef(output->handlers); set_name_table(p, p->top); + p->ignore_json_unknown = ignore_json_unknown; + /* If this fails, uncomment and increase the value in parser.h. */ /* fprintf(stderr, "%zd\n", upb_env_bytesallocated(env) - size_before); */ UPB_ASSERT_DEBUGVAR(upb_env_bytesallocated(env) - size_before <= diff --git a/php/ext/google/protobuf/upb.h b/php/ext/google/protobuf/upb.h index a263db30..180f4715 100644 --- a/php/ext/google/protobuf/upb.h +++ b/php/ext/google/protobuf/upb.h @@ -9457,7 +9457,7 @@ UPB_DECLARE_DERIVED_TYPE(upb::json::ParserMethod, upb::RefCounted, class upb::json::Parser { public: static Parser* Create(Environment* env, const ParserMethod* method, - Sink* output); + Sink* output, bool ignore_json_unknown); BytesSink* input(); @@ -9491,7 +9491,8 @@ UPB_BEGIN_EXTERN_C upb_json_parser* upb_json_parser_create(upb_env* e, const upb_json_parsermethod* m, - upb_sink* output); + upb_sink* output, + bool ignore_json_unknown); upb_bytessink *upb_json_parser_input(upb_json_parser *p); upb_json_parsermethod* upb_json_parsermethod_new(const upb_msgdef* md, @@ -9511,8 +9512,8 @@ UPB_END_EXTERN_C namespace upb { namespace json { inline Parser* Parser::Create(Environment* env, const ParserMethod* method, - Sink* output) { - return upb_json_parser_create(env, method, output); + Sink* output, bool ignore_json_unknown) { + return upb_json_parser_create(env, method, output, ignore_json_unknown); } inline BytesSink* Parser::input() { return upb_json_parser_input(this); diff --git a/php/src/Google/Protobuf/Internal/Descriptor.php b/php/src/Google/Protobuf/Internal/Descriptor.php index ee3a8bde..21ac5de3 100644 --- a/php/src/Google/Protobuf/Internal/Descriptor.php +++ b/php/src/Google/Protobuf/Internal/Descriptor.php @@ -44,6 +44,7 @@ class Descriptor private $nested_type = []; private $enum_type = []; private $klass; + private $legacy_klass; private $options; private $oneof_decl = []; @@ -151,6 +152,16 @@ class Descriptor return $this->klass; } + public function setLegacyClass($klass) + { + $this->legacy_klass = $klass; + } + + public function getLegacyClass() + { + return $this->legacy_klass; + } + public function setOptions($options) { $this->options = $options; @@ -167,16 +178,19 @@ class Descriptor $message_name_without_package = ""; $classname = ""; + $legacy_classname = ""; $fullname = ""; GPBUtil::getFullClassName( $proto, $containing, $file_proto, $message_name_without_package, + $legacy_classname, $classname, $fullname); $desc->setFullName($fullname); $desc->setClass($classname); + $desc->setLegacyClass($legacy_classname); $desc->setOptions($proto->getOptions()); foreach ($proto->getField() as $field_proto) { diff --git a/php/src/Google/Protobuf/Internal/DescriptorPool.php b/php/src/Google/Protobuf/Internal/DescriptorPool.php index 304c1615..9b4dcc01 100644 --- a/php/src/Google/Protobuf/Internal/DescriptorPool.php +++ b/php/src/Google/Protobuf/Internal/DescriptorPool.php @@ -92,6 +92,7 @@ class DescriptorPool $this->proto_to_class[$descriptor->getFullName()] = $descriptor->getClass(); $this->class_to_desc[$descriptor->getClass()] = $descriptor; + $this->class_to_desc[$descriptor->getLegacyClass()] = $descriptor; foreach ($descriptor->getNestedType() as $nested_type) { $this->addDescriptor($nested_type); } @@ -105,6 +106,7 @@ class DescriptorPool $this->proto_to_class[$descriptor->getFullName()] = $descriptor->getClass(); $this->class_to_enum_desc[$descriptor->getClass()] = $descriptor; + $this->class_to_enum_desc[$descriptor->getLegacyClass()] = $descriptor; } public function getDescriptorByClassName($klass) diff --git a/php/src/Google/Protobuf/Internal/EnumDescriptor.php b/php/src/Google/Protobuf/Internal/EnumDescriptor.php index 01649fec..82a42767 100644 --- a/php/src/Google/Protobuf/Internal/EnumDescriptor.php +++ b/php/src/Google/Protobuf/Internal/EnumDescriptor.php @@ -9,6 +9,7 @@ class EnumDescriptor use HasPublicDescriptorTrait; private $klass; + private $legacy_klass; private $full_name; private $value; private $name_to_value; @@ -66,12 +67,23 @@ class EnumDescriptor return $this->klass; } + public function setLegacyClass($klass) + { + $this->legacy_klass = $klass; + } + + public function getLegacyClass() + { + return $this->legacy_klass; + } + public static function buildFromProto($proto, $file_proto, $containing) { $desc = new EnumDescriptor(); $enum_name_without_package = ""; $classname = ""; + $legacy_classname = ""; $fullname = ""; GPBUtil::getFullClassName( $proto, @@ -79,9 +91,11 @@ class EnumDescriptor $file_proto, $enum_name_without_package, $classname, + $legacy_classname, $fullname); $desc->setFullName($fullname); $desc->setClass($classname); + $desc->setLegacyClass($legacy_classname); $values = $proto->getValue(); foreach ($values as $value) { $desc->addValue($value->getNumber(), $value); diff --git a/php/src/Google/Protobuf/Internal/GPBJsonWire.php b/php/src/Google/Protobuf/Internal/GPBJsonWire.php index 9ae57ab3..43f4745d 100644 --- a/php/src/Google/Protobuf/Internal/GPBJsonWire.php +++ b/php/src/Google/Protobuf/Internal/GPBJsonWire.php @@ -226,7 +226,7 @@ class GPBJsonWire $output->writeRaw("\"", 1); break; case GPBType::STRING: - $value = json_encode($value); + $value = json_encode($value, JSON_UNESCAPED_UNICODE); $output->writeRaw($value, strlen($value)); break; // case GPBType::GROUP: diff --git a/php/src/Google/Protobuf/Internal/GPBUtil.php b/php/src/Google/Protobuf/Internal/GPBUtil.php index 023b07f4..ec0bf6bd 100644 --- a/php/src/Google/Protobuf/Internal/GPBUtil.php +++ b/php/src/Google/Protobuf/Internal/GPBUtil.php @@ -215,9 +215,10 @@ class GPBUtil "Expect repeated field of different type."); } if ($var->getType() === GPBType::MESSAGE && - $var->getClass() !== $klass) { + $var->getClass() !== $klass && + $var->getLegacyClass() !== $klass) { throw new \Exception( - "Expect repeated field of different message."); + "Expect repeated field of " . $klass . "."); } return $var; } @@ -242,9 +243,10 @@ class GPBUtil throw new \Exception("Expect map field of value type."); } if ($var->getValueType() === GPBType::MESSAGE && - $var->getValueClass() !== $klass) { + $var->getValueClass() !== $klass && + $var->getLegacyValueClass() !== $klass) { throw new \Exception( - "Expect map field of different value message."); + "Expect map field of " . $klass . "."); } return $var; } @@ -299,6 +301,14 @@ class GPBUtil return ""; } + public static function getLegacyClassNameWithoutPackage( + $name, + $file_proto) + { + $classname = implode('_', explode('.', $name)); + return static::getClassNamePrefix($classname, $file_proto) . $classname; + } + public static function getClassNameWithoutPackage( $name, $file_proto) @@ -316,6 +326,7 @@ class GPBUtil $file_proto, &$message_name_without_package, &$classname, + &$legacy_classname, &$fullname) { // Full name needs to start with '.'. @@ -334,21 +345,28 @@ class GPBUtil $class_name_without_package = static::getClassNameWithoutPackage($message_name_without_package, $file_proto); + $legacy_class_name_without_package = + static::getLegacyClassNameWithoutPackage( + $message_name_without_package, $file_proto); $option = $file_proto->getOptions(); if (!is_null($option) && $option->hasPhpNamespace()) { $namespace = $option->getPhpNamespace(); if ($namespace !== "") { $classname = $namespace . "\\" . $class_name_without_package; + $legacy_classname = + $namespace . "\\" . $legacy_class_name_without_package; return; } else { $classname = $class_name_without_package; + $legacy_classname = $legacy_class_name_without_package; return; } } if ($package === "") { $classname = $class_name_without_package; + $legacy_classname = $legacy_class_name_without_package; } else { $parts = array_map('ucwords', explode('.', $package)); foreach ($parts as $i => $part) { @@ -358,6 +376,9 @@ class GPBUtil implode('\\', $parts) . "\\".self::getClassNamePrefix($class_name_without_package,$file_proto). $class_name_without_package; + $legacy_classname = + implode('\\', array_map('ucwords', explode('.', $package))). + "\\".$legacy_class_name_without_package; } } diff --git a/php/src/Google/Protobuf/Internal/MapField.php b/php/src/Google/Protobuf/Internal/MapField.php index 38736dad..1b2b0cf0 100644 --- a/php/src/Google/Protobuf/Internal/MapField.php +++ b/php/src/Google/Protobuf/Internal/MapField.php @@ -58,7 +58,11 @@ class MapField implements \ArrayAccess, \IteratorAggregate, \Countable /** * @ignore */ - private $value_klass; + private $klass; + /** + * @ignore + */ + private $legacy_klass; /** * Constructs an instance of MapField. @@ -75,6 +79,17 @@ class MapField implements \ArrayAccess, \IteratorAggregate, \Countable $this->key_type = $key_type; $this->value_type = $value_type; $this->klass = $klass; + + if ($this->value_type == GPBType::MESSAGE) { + $pool = DescriptorPool::getGeneratedPool(); + $desc = $pool->getDescriptorByClassName($klass); + if ($desc == NULL) { + new $klass; // No msg class instance has been created before. + $desc = $pool->getDescriptorByClassName($klass); + } + $this->klass = $desc->getClass(); + $this->legacy_klass = $desc->getLegacyClass(); + } } /** @@ -102,6 +117,14 @@ class MapField implements \ArrayAccess, \IteratorAggregate, \Countable } /** + * @ignore + */ + public function getLegacyValueClass() + { + return $this->legacy_klass; + } + + /** * Return the element at the given key. * * This will also be called for: $ele = $arr[$key] @@ -133,15 +156,22 @@ class MapField implements \ArrayAccess, \IteratorAggregate, \Countable $this->checkKey($this->key_type, $key); switch ($this->value_type) { + case GPBType::SFIXED32: + case GPBType::SINT32: case GPBType::INT32: + case GPBType::ENUM: GPBUtil::checkInt32($value); break; + case GPBType::FIXED32: case GPBType::UINT32: GPBUtil::checkUint32($value); break; + case GPBType::SFIXED64: + case GPBType::SINT64: case GPBType::INT64: GPBUtil::checkInt64($value); break; + case GPBType::FIXED64: case GPBType::UINT64: GPBUtil::checkUint64($value); break; @@ -226,36 +256,24 @@ class MapField implements \ArrayAccess, \IteratorAggregate, \Countable private function checkKey($key_type, &$key) { switch ($key_type) { + case GPBType::SFIXED32: + case GPBType::SINT32: case GPBType::INT32: GPBUtil::checkInt32($key); break; + case GPBType::FIXED32: case GPBType::UINT32: GPBUtil::checkUint32($key); break; + case GPBType::SFIXED64: + case GPBType::SINT64: case GPBType::INT64: GPBUtil::checkInt64($key); break; - case GPBType::UINT64: - GPBUtil::checkUint64($key); - break; case GPBType::FIXED64: + case GPBType::UINT64: GPBUtil::checkUint64($key); break; - case GPBType::FIXED32: - GPBUtil::checkUint32($key); - break; - case GPBType::SFIXED64: - GPBUtil::checkInt64($key); - break; - case GPBType::SFIXED32: - GPBUtil::checkInt32($key); - break; - case GPBType::SINT64: - GPBUtil::checkInt64($key); - break; - case GPBType::SINT32: - GPBUtil::checkInt32($key); - break; case GPBType::BOOL: GPBUtil::checkBool($key); break; diff --git a/php/src/Google/Protobuf/Internal/Message.php b/php/src/Google/Protobuf/Internal/Message.php index 73ac375e..8ff141ca 100644 --- a/php/src/Google/Protobuf/Internal/Message.php +++ b/php/src/Google/Protobuf/Internal/Message.php @@ -1563,7 +1563,7 @@ class Message } break; case GPBType::STRING: - $value = json_encode($value); + $value = json_encode($value, JSON_UNESCAPED_UNICODE); $size += strlen($value); break; case GPBType::BYTES: diff --git a/php/src/Google/Protobuf/Internal/RepeatedField.php b/php/src/Google/Protobuf/Internal/RepeatedField.php index 797b3b3a..e9b92d8d 100644 --- a/php/src/Google/Protobuf/Internal/RepeatedField.php +++ b/php/src/Google/Protobuf/Internal/RepeatedField.php @@ -59,6 +59,10 @@ class RepeatedField implements \ArrayAccess, \IteratorAggregate, \Countable * @ignore */ private $klass; + /** + * @ignore + */ + private $legacy_klass; /** * Constructs an instance of RepeatedField. @@ -71,7 +75,16 @@ class RepeatedField implements \ArrayAccess, \IteratorAggregate, \Countable { $this->container = []; $this->type = $type; - $this->klass = $klass; + if ($this->type == GPBType::MESSAGE) { + $pool = DescriptorPool::getGeneratedPool(); + $desc = $pool->getDescriptorByClassName($klass); + if ($desc == NULL) { + new $klass; // No msg class instance has been created before. + $desc = $pool->getDescriptorByClassName($klass); + } + $this->klass = $desc->getClass(); + $this->legacy_klass = $desc->getLegacyClass(); + } } /** @@ -91,6 +104,14 @@ class RepeatedField implements \ArrayAccess, \IteratorAggregate, \Countable } /** + * @ignore + */ + public function getLegacyClass() + { + return $this->legacy_klass; + } + + /** * Return the element at the given index. * * This will also be called for: $ele = $arr[0] @@ -120,15 +141,22 @@ class RepeatedField implements \ArrayAccess, \IteratorAggregate, \Countable public function offsetSet($offset, $value) { switch ($this->type) { + case GPBType::SFIXED32: + case GPBType::SINT32: case GPBType::INT32: + case GPBType::ENUM: GPBUtil::checkInt32($value); break; + case GPBType::FIXED32: case GPBType::UINT32: GPBUtil::checkUint32($value); break; + case GPBType::SFIXED64: + case GPBType::SINT64: case GPBType::INT64: GPBUtil::checkInt64($value); break; + case GPBType::FIXED64: case GPBType::UINT64: GPBUtil::checkUint64($value); break; @@ -141,6 +169,9 @@ class RepeatedField implements \ArrayAccess, \IteratorAggregate, \Countable case GPBType::BOOL: GPBUtil::checkBool($value); break; + case GPBType::BYTES: + GPBUtil::checkString($value, false); + break; case GPBType::STRING: GPBUtil::checkString($value, true); break; diff --git a/php/tests/compatibility_test.sh b/php/tests/compatibility_test.sh index 7caa46e7..b377d85c 100755 --- a/php/tests/compatibility_test.sh +++ b/php/tests/compatibility_test.sh @@ -2,12 +2,14 @@ function use_php() { VERSION=$1 - PHP=`which php` - PHP_CONFIG=`which php-config` - PHPIZE=`which phpize` - ln -sfn "/usr/local/php-${VERSION}/bin/php" $PHP - ln -sfn "/usr/local/php-${VERSION}/bin/php-config" $PHP_CONFIG - ln -sfn "/usr/local/php-${VERSION}/bin/phpize" $PHPIZE + + OLD_PATH=$PATH + OLD_CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH + OLD_C_INCLUDE_PATH=$C_INCLUDE_PATH + + export PATH=/usr/local/php-${VERSION}/bin:$OLD_PATH + export CPLUS_INCLUDE_PATH=/usr/local/php-${VERSION}/include/php/main:/usr/local/php-${VERSION}/include/php/:$OLD_CPLUS_INCLUDE_PATH + export C_INCLUDE_PATH=/usr/local/php-${VERSION}/include/php/main:/usr/local/php-${VERSION}/include/php/:$OLD_C_INCLUDE_PATH } function generate_proto() { @@ -18,7 +20,22 @@ function generate_proto() { mkdir generated $PROTOC1 --php_out=generated proto/test_include.proto - $PROTOC2 --php_out=generated proto/test.proto proto/test_no_namespace.proto proto/test_prefix.proto + $PROTOC2 --php_out=generated \ + proto/test.proto \ + proto/test_no_namespace.proto \ + proto/test_prefix.proto \ + proto/test_php_namespace.proto \ + proto/test_empty_php_namespace.proto \ + proto/test_reserved_enum_lower.proto \ + proto/test_reserved_enum_upper.proto \ + proto/test_reserved_enum_value_lower.proto \ + proto/test_reserved_enum_value_upper.proto \ + proto/test_reserved_message_lower.proto \ + proto/test_reserved_message_upper.proto \ + proto/test_service.proto \ + proto/test_service_namespace.proto \ + proto/test_descriptors.proto + pushd ../../src $PROTOC2 --php_out=../php/tests/generated -I../php/tests -I. ../php/tests/proto/test_import_descriptor_proto.proto popd @@ -52,9 +69,9 @@ cd $(dirname $0) # The old version of protobuf that we are testing compatibility against. case "$1" in - ""|3.3.0) - OLD_VERSION=3.3.0 - OLD_VERSION_PROTOC=http://repo1.maven.org/maven2/com/google/protobuf/protoc/3.3.0/protoc-3.3.0-linux-x86_64.exe + ""|3.5.0) + OLD_VERSION=3.5.0 + OLD_VERSION_PROTOC=http://repo1.maven.org/maven2/com/google/protobuf/protoc/$OLD_VERSION/protoc-$OLD_VERSION-linux-x86_64.exe ;; *) echo "[ERROR]: Unknown version number: $1" @@ -81,7 +98,7 @@ git checkout v$OLD_VERSION popd # Build and copy the new runtime -use_php 5.5 +use_php 7.1 pushd ../ext/google/protobuf make clean || true phpize && ./configure && make @@ -99,12 +116,14 @@ chmod +x old_protoc NEW_PROTOC=`pwd`/../../src/protoc OLD_PROTOC=`pwd`/old_protoc cd protobuf/php -cp -r /usr/local/vendor-5.5 vendor -wget https://phar.phpunit.de/phpunit-4.8.0.phar -O /usr/bin/phpunit +composer install # Remove implementation detail tests. tests=( array_test.php encode_decode_test.php generated_class_test.php map_field_test.php well_known_test.php ) sed -i.bak '/php_implementation_test.php/d' phpunit.xml +sed -i.bak '/generated_phpdoc_test.php/d' phpunit.xml +sed -i.bak 's/generated_phpdoc_test.php//g' tests/test.sh +sed -i.bak '/memory_leak_test.php/d' tests/test.sh for t in "${tests[@]}" do remove_error_test tests/$t @@ -118,7 +137,7 @@ cd tests generate_proto $OLD_PROTOC $OLD_PROTOC ./test.sh pushd .. -phpunit +./vendor/bin/phpunit popd # Test A.2: @@ -127,7 +146,7 @@ popd generate_proto $NEW_PROTOC $OLD_PROTOC ./test.sh pushd .. -phpunit +./vendor/bin/phpunit popd # Test A.3: @@ -136,5 +155,5 @@ popd generate_proto $OLD_PROTOC $NEW_PROTOC ./test.sh pushd .. -phpunit +./vendor/bin/phpunit popd diff --git a/php/tests/test.sh b/php/tests/test.sh index 6e70eb2a..700dd295 100755 --- a/php/tests/test.sh +++ b/php/tests/test.sh @@ -14,7 +14,7 @@ set -e phpize && ./configure CFLAGS='-g -O0' && make popd -tests=( array_test.php encode_decode_test.php generated_class_test.php generated_phpdoc_test.php map_field_test.php well_known_test.php generated_service_test.php descriptors_test.php ) +tests=( array_test.php encode_decode_test.php generated_class_test.php map_field_test.php well_known_test.php descriptors_test.php ) for t in "${tests[@]}" do diff --git a/post_process_dist.sh b/post_process_dist.sh index eb5f584d..5b14de67 100755 --- a/post_process_dist.sh +++ b/post_process_dist.sh @@ -27,7 +27,7 @@ fi set -ex -LANGUAGES="cpp csharp java javanano js objectivec python ruby php all" +LANGUAGES="cpp csharp java js objectivec python ruby php all" BASENAME=`basename $1 .tar.gz` VERSION=${BASENAME:9} diff --git a/protobuf.bzl b/protobuf.bzl index 8bbd0564..67d61d7b 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -74,7 +74,7 @@ def _proto_gen_impl(ctx): deps = [] deps += ctx.files.srcs source_dir = _SourceDir(ctx) - gen_dir = _GenDir(ctx) + gen_dir = _GenDir(ctx).rstrip('/') if source_dir: import_flags = ["-I" + source_dir, "-I" + gen_dir] else: @@ -84,37 +84,83 @@ def _proto_gen_impl(ctx): import_flags += dep.proto.import_flags deps += dep.proto.deps - args = [] - if ctx.attr.gen_cc: - args += ["--cpp_out=" + gen_dir] - if ctx.attr.gen_py: - args += ["--python_out=" + gen_dir] - - inputs = srcs + deps - if ctx.executable.plugin: - plugin = ctx.executable.plugin - lang = ctx.attr.plugin_language - if not lang and plugin.basename.startswith('protoc-gen-'): - lang = plugin.basename[len('protoc-gen-'):] - if not lang: - fail("cannot infer the target language of plugin", "plugin_language") - - outdir = gen_dir - if ctx.attr.plugin_options: - outdir = ",".join(ctx.attr.plugin_options) + ":" + outdir - args += ["--plugin=protoc-gen-%s=%s" % (lang, plugin.path)] - args += ["--%s_out=%s" % (lang, outdir)] - inputs += [plugin] - - if args: - ctx.action( - inputs=inputs, - outputs=ctx.outputs.outs, - arguments=args + import_flags + [s.path for s in srcs], - executable=ctx.executable.protoc, - mnemonic="ProtoCompile", - use_default_shell_env=True, - ) + if not ctx.attr.gen_cc and not ctx.attr.gen_py and not ctx.executable.plugin: + return struct( + proto=struct( + srcs=srcs, + import_flags=import_flags, + deps=deps, + ), + ) + + for src in srcs: + args = [] + + in_gen_dir = src.root.path == gen_dir + if in_gen_dir: + import_flags_real = [] + for f in depset(import_flags): + path = f.replace('-I', '') + import_flags_real.append('-I$(realpath -s %s)' % path) + + outs = [] + use_grpc_plugin = (ctx.attr.plugin_language == "grpc" and ctx.attr.plugin) + path_tpl = "$(realpath %s)" if in_gen_dir else "%s" + if ctx.attr.gen_cc: + args += [("--cpp_out=" + path_tpl) % gen_dir] + outs.extend(_CcOuts([src.basename], use_grpc_plugin=use_grpc_plugin)) + if ctx.attr.gen_py: + args += [("--python_out=" + path_tpl) % gen_dir] + outs.extend(_PyOuts([src.basename], use_grpc_plugin=use_grpc_plugin)) + + outs = [ctx.actions.declare_file(out, sibling=src) for out in outs] + inputs = [src] + deps + if ctx.executable.plugin: + plugin = ctx.executable.plugin + lang = ctx.attr.plugin_language + if not lang and plugin.basename.startswith('protoc-gen-'): + lang = plugin.basename[len('protoc-gen-'):] + if not lang: + fail("cannot infer the target language of plugin", "plugin_language") + + outdir = "." if in_gen_dir else gen_dir + + if ctx.attr.plugin_options: + outdir = ",".join(ctx.attr.plugin_options) + ":" + outdir + args += [("--plugin=protoc-gen-%s=" + path_tpl) % (lang, plugin.path)] + args += ["--%s_out=%s" % (lang, outdir)] + inputs += [plugin] + + if not in_gen_dir: + ctx.action( + inputs=inputs, + outputs=outs, + arguments=args + import_flags + [src.path], + executable=ctx.executable.protoc, + mnemonic="ProtoCompile", + use_default_shell_env=True, + ) + else: + for out in outs: + orig_command = " ".join( + ["$(realpath %s)" % ctx.executable.protoc.path] + args + + import_flags_real + ["-I.", src.basename]) + command = ";".join([ + 'CMD="%s"' % orig_command, + "cd %s" % src.dirname, + "${CMD}", + "cd -", + ]) + generated_out = '/'.join([gen_dir, out.basename]) + if generated_out != out.path: + command += ";mv %s %s" % (generated_out, out.path) + ctx.action( + inputs=inputs + [ctx.executable.protoc], + outputs=[out], + command=command, + mnemonic="ProtoCompile", + use_default_shell_env=True, + ) return struct( proto=struct( diff --git a/protoc-artifacts/Dockerfile b/protoc-artifacts/Dockerfile index c346586b..95086eae 100644 --- a/protoc-artifacts/Dockerfile +++ b/protoc-artifacts/Dockerfile @@ -1,4 +1,4 @@ -FROM centos:6.6 +FROM centos:6.9 RUN yum install -y git \ tar \ @@ -28,15 +28,14 @@ RUN wget -q http://apache.cs.utah.edu/maven/maven-3/3.3.9/binaries/apache-maven- tar xz -C /var/local ENV PATH /var/local/apache-maven-3.3.9/bin:$PATH -# Install GCC 4.7 to support -static-libstdc++ -RUN wget http://people.centos.org/tru/devtools-1.1/devtools-1.1.repo -P /etc/yum.repos.d && \ - bash -c 'echo "enabled=1" >> /etc/yum.repos.d/devtools-1.1.repo' && \ - bash -c "sed -e 's/\$basearch/i386/g' /etc/yum.repos.d/devtools-1.1.repo > /etc/yum.repos.d/devtools-i386-1.1.repo" && \ - sed -e 's/testing-/testing-i386-/g' -i /etc/yum.repos.d/devtools-i386-1.1.repo && \ +# Install GCC 4.8 to support -static-libstdc++ +RUN wget http://people.centos.org/tru/devtools-2/devtools-2.repo -P /etc/yum.repos.d && \ + bash -c 'echo "enabled=1" >> /etc/yum.repos.d/devtools-2.repo' && \ + bash -c "sed -e 's/\$basearch/i386/g' /etc/yum.repos.d/devtools-2.repo > /etc/yum.repos.d/devtools-i386-2.repo" && \ + sed -e 's/testing-/testing-i386-/g' -i /etc/yum.repos.d/devtools-i386-2.repo && \ rpm --rebuilddb && \ - yum install -y devtoolset-1.1 \ - devtoolset-1.1-libstdc++-devel \ - devtoolset-1.1-libstdc++-devel.i686 && \ + yum install -y devtoolset-2-gcc devtoolset-2-gcc-c++ devtoolset-2-binutils devtoolset-2-libstdc++-devel \ + devtoolset-2-gcc.i686 devtoolset-2-gcc-c++.i686 devtoolset-2-binutils.i686 devtoolset-2-libstdc++-devel.i686 && \ yum clean all COPY scl-enable-devtoolset.sh /var/local/ diff --git a/protoc-artifacts/README.md b/protoc-artifacts/README.md index dcaec987..c25406dd 100644 --- a/protoc-artifacts/README.md +++ b/protoc-artifacts/README.md @@ -7,13 +7,31 @@ build and publish a ``protoc`` executable (a.k.a. artifact) to Maven repositories. The artifact can be used by build automation tools so that users would not need to compile and install ``protoc`` for their systems. +If you would like us to publish protoc artifact for a new platform, please send +us a pull request to add support for the new platform. You would need to change +the following files: + +* [build-protoc.sh](build-protoc.sh): script to cross-build the protoc for your + platform. +* [pom.xml](pom.xml): script to upload artifacts to maven. +* [build-zip.sh](build-zip.sh): script to package published maven artifacts in + our release page. + +## Maven Location +The published protoc artifacts are available on Maven here: + + http://central.maven.org/maven2/com/google/protobuf/protoc/ + ## Versioning The version of the ``protoc`` artifact must be the same as the version of the Protobuf project. ## Artifact name The name of a published ``protoc`` artifact is in the following format: -``protoc-<version>-<os>-<arch>.exe``, e.g., ``protoc-3.0.0-alpha-3-windows-x86_64.exe``. +``protoc-<version>-<os>-<arch>.exe``, e.g., ``protoc-3.6.1-linux-x86_64.exe``. + +Note that artifacts for linux/macos also have the `.exe` suffix but they are +not windows binaries. ## System requirement Install [Apache Maven](http://maven.apache.org/) if you don't have it. @@ -29,95 +47,116 @@ generate the configure script. Under the protobuf project directory: + ``` -$ ./autogen.sh && ./configure && make +$ ./autogen.sh ``` -## To install artifacts locally -The following command will install the ``protoc`` artifact to your local Maven repository. +### Build the artifact for each platform + +Run the build-protoc.sh script under this protoc-artifacts directory to build the protoc +artifact for each platform. For example: + ``` -$ mvn install +$ cd protoc-artifacts +$ ./build-protoc.sh linux x86_64 protoc ``` -## Cross-compilation -The Maven script will try to detect the OS and the architecture from Java -system properties. It's possible to build a protoc binary for an architecture -that is different from what Java has detected, as long as you have the proper -compilers installed. +The above command will produce a `target/linux/x86_64/protoc` binary under the +protoc-artifacts directory. -You can override the Maven properties ``os.detected.name`` and -``os.detected.arch`` to force the script to generate binaries for a specific OS -and/or architecture. Valid values are defined as the return values of -``normalizeOs()`` and ``normalizeArch()`` of ``Detector`` from -[os-maven-plugin](https://github.com/trustin/os-maven-plugin/blob/master/src/main/java/kr/motd/maven/os/Detector.java). -Frequently used values are: -- ``os.detected.name``: ``linux``, ``osx``, ``windows``. -- ``os.detected.arch``: ``x86_32``, ``x86_64`` +For a list of supported platforms, see the comments in the build-protoc.sh +script. We only use this script to build artifacts on Ubuntu and MacOS (both +with x86_64, and do cross-compilation for other platforms. + +### Tips for building for Linux +We build on Centos 6.9 to provide a good compatibility for not very new +systems. We have provided a ``Dockerfile`` under this directory to build the +environment. It has been tested with Docker 1.6.1. + +To build a image: -For example, MinGW32 only ships with 32-bit compilers, but you can still build -32-bit protoc under 64-bit Windows, with the following command: ``` -$ mvn install -Dos.detected.arch=x86_32 +$ docker build -t protoc-artifacts . ``` +To run the image: + +``` +$ docker run -it --rm=true protoc-artifacts bash +``` + +To checkout protobuf (run within the container): + +``` +$ # Replace v3.5.1 with the version you want +$ wget -O - https://github.com/google/protobuf/archive/v3.5.1.tar.gz | tar xvzp +``` + +### Windows build +We no longer use scripts in this directory to build windows artifacts. Instead, +we use Visual Studio 2015 to build our windows release artifacts. See our +[kokoro windows build scripts here](../kokoro/release/protoc/windows/build.bat). + +To upload windows artifacts, copy the built binaries into this directory and +put it into the target/windows/(x86_64|x86_32) directory the same way as the +artifacts for other platforms. That will allow the maven script to find and +upload the artifacts to maven. + ## To push artifacts to Maven Central Before you can upload artifacts to Maven Central repository, make sure you have read [this page](http://central.sonatype.org/pages/apache-maven.html) on how to configure GPG and Sonatype account. -You need to perform the deployment for every platform that you want to -support. DO NOT close the staging repository until you have done the -deployment for all platforms. Currently the following platforms are supported: -- Linux (x86_32, x86_64 and cross compiled aarch_64) -- Windows (x86_32 and x86_64) with - - Cygwin64 with MinGW compilers (x86_64) - - MSYS with MinGW32 (x86_32) - - Cross compile in Linux with MinGW-w64 (x86_32, x86_64) -- MacOSX (x86_32 and x86_64) - -As for MSYS2/MinGW64 for Windows: protoc will build, but it insists on -adding a dependency of `libwinpthread-1.dll`, which isn't shipped with -Windows. +Before you do the deployment, make sure you have built the protoc artifacts for +every supported platform and put them under the target directory. Example +target directory layout: + + + pom.xml + + target + + linux + + x86_64 + protoc.exe + + x86_32 + protoc.exe + + macos + + x86_64 + protoc.exe + + x86_32 + protoc.exe + + windows + + x86_64 + protoc.exe + + x86_32 + protoc.exe + +You will need to build the artifacts on multiple machines and gather them +together into one place. Use the following command to deploy artifacts for the host platform to a staging repository. + ``` $ mvn clean deploy -P release ``` + It creates a new staging repository. Go to https://oss.sonatype.org/#stagingRepositories and find the repository, usually -in the name like ``comgoogle-123``. - -You will want to run this command on a different platform. Remember, in -subsequent deployments you will need to provide the repository name that you -have found in the first deployment so that all artifacts go to the same -repository: -``` -$ mvn clean deploy -P release -Dstaging.repository=comgoogle-123 -``` - -A 32-bit artifact can be deployed from a 64-bit host with -``-Dos.detected.arch=x86_32`` - -An arm64 artifact can be deployed from x86 host with -``-Dos.detected.arch=aarch_64`` - -A windows artifact can be deployed from a linux machine with -``-Dos.detected.name=windows`` - -When you have done deployment for all platforms, go to -https://oss.sonatype.org/#stagingRepositories, verify that the staging -repository has all the binaries, close and release this repository. +in the name like ``comgoogle-123``. Verify that the staging repository has all +the binaries, close and release this repository. ## Upload zip packages to github release page. After uploading protoc artifacts to Maven Central repository, run the build-zip.sh script to bulid zip packages for these protoc binaries and upload these zip packages to the download section of the github release. For example: + ``` -$ ./build-zip.sh 3.0.0-beta-4 +$ ./build-zip.sh 3.6.0 ``` + The above command will create 5 zip files: + ``` dist/protoc-3.0.0-beta-4-win32.zip dist/protoc-3.0.0-beta-4-osx-x86_32.zip @@ -125,67 +164,14 @@ dist/protoc-3.0.0-beta-4-osx-x86_64.zip dist/protoc-3.0.0-beta-4-linux-x86_32.zip dist/protoc-3.0.0-beta-4-linux-x86_64.zip ``` + Before running the script, make sure the artifacts are accessible from: http://repo1.maven.org/maven2/com/google/protobuf/protoc/ -### Tips for deploying on Linux -We build on Centos 6.6 to provide a good compatibility for not very new -systems. We have provided a ``Dockerfile`` under this directory to build the -environment. It has been tested with Docker 1.6.1. - -To build a image: -``` -$ docker build -t protoc-artifacts . -``` - -To run the image: -``` -$ docker run -it --rm=true protoc-artifacts bash -``` - -To checkout protobuf (run within the container): -``` -$ # Replace v3.5.1 with the version you want -$ wget -O - https://github.com/google/protobuf/archive/v3.5.1.tar.gz | tar xvzp -``` - -### Tips for deploying on Windows -Under Windows the following error may occur: ``gpg: cannot open tty `no tty': -No such file or directory``. This can be fixed by configuring gpg through an -active profile in ``.m2\settings.xml`` where also the Sonatype password is -stored: -```xml -<settings> - <servers> - <server> - <id>sonatype-nexus-staging</id> - <username>[username]</username> - <password>[password]</password> - </server> - </servers> - <profiles> - <profile> - <id>gpg</id> - <properties> - <gpg.executable>gpg</gpg.executable> - <gpg.passphrase>[password]</gpg.passphrase> - </properties> - </profile> - </profiles> - <activeProfiles> - <activeProfile>gpg</activeProfile> - </activeProfiles> -</settings> -``` - -### Tested build environments +## Tested build environments We have successfully built artifacts on the following environments: - Linux x86_32 and x86_64: - - Centos 6.6 (within Docker 1.6.1) - - Ubuntu 14.04.2 64-bit -- Linux aarch_64: Cross compiled with `g++-aarch64-linux-gnu` on Ubuntu 14.04.2 64-bit -- Windows x86_32: MSYS with ``mingw32-gcc-g++ 4.8.1-4`` on Windows 7 64-bit -- Windows x86_32: Cross compile with ``i686-w64-mingw32-g++ 4.8.2`` on Ubuntu 14.04.2 64-bit -- Windows x86_64: Cygwin64 with ``mingw64-x86_64-gcc-g++ 4.8.3-1`` on Windows 7 64-bit -- Windows x86_64: Cross compile with ``x86_64-w64-mingw32-g++ 4.8.2`` on Ubuntu 14.04.2 64-bit + - Centos 6.9 (within Docker 1.6.1) + - Ubuntu 14.04.5 64-bit +- Linux aarch_64: Cross compiled with `g++-aarch64-linux-gnu` on Ubuntu 14.04.5 64-bit - Mac OS X x86_32 and x86_64: Mac OS X 10.9.5 diff --git a/protoc-artifacts/build-protoc.sh b/protoc-artifacts/build-protoc.sh index b78abbcf..d6a3cfa3 100755 --- a/protoc-artifacts/build-protoc.sh +++ b/protoc-artifacts/build-protoc.sh @@ -1,23 +1,34 @@ #!/bin/bash -# Builds protoc executable into target/protoc.exe; optionally build protoc -# plugins into target/protoc-gen-*.exe -# To be run from Maven. -# Usage: build-protoc.sh <OS> <ARCH> <TARGET> -# <OS> and <ARCH> are ${os.detected.name} and ${os.detected.arch} from os-maven-plugin -# <TARGET> can be "protoc" or "protoc-gen-javalite" +# Builds protoc executable into target/<OS>/<ARCH>/protoc.exe; optionally builds +# protoc plugins into target/<OS>/<ARCH>/protoc-gen-*.exe # -# The script now supports cross-compiling windows and linux-arm64 in linux-x86 -# environment. Required packages: -# - Windows: i686-w64-mingw32-gcc (32bit) and x86_64-w64-mingw32-gcc (64bit) -# - Arm64: g++-aarch64-linux-gnu +# Usage: ./build-protoc.sh <OS> <ARCH> <TARGET> +# +# <TARGET> can be "protoc" or "protoc-gen-javalite". Supported <OS> <ARCH> +# combinations: +# HOST <OS> <ARCH> <COMMENT> +# cygwin windows x86_32 Requires: i686-w64-mingw32-gcc +# cygwin windows x86_64 Requires: x86_64-w64-mingw32-gcc +# linux linux aarch_64 Requires: g++-aarch64-linux-gnu +# linux linux x86_32 +# linux linux x86_64 +# linux windows x86_32 Requires: i686-w64-mingw32-gcc +# linux windows x86_64 Requires: x86_64-w64-mingw32-gcc +# macos osx x86_32 +# macos osx x86_64 +# mingw windows x86_32 +# mingw windows x86_64 +# +# Before running this script, make sure you have generated the configure script +# in the parent directory (i.e., run ./autogen.sh there). OS=$1 ARCH=$2 MAKE_TARGET=$3 if [[ $# < 3 ]]; then - echo "No arguments provided. This script is intended to be run from Maven." + echo "Not enough arguments provided." exit 1 fi @@ -27,7 +38,7 @@ case $MAKE_TARGET in protoc) ;; *) - echo "Target ""$TARGET"" invalid." + echo "Target ""$MAKE_TARGET"" invalid." exit 1 esac @@ -130,7 +141,7 @@ checkDependencies () white_list="linux-vdso64\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|libz\.so\.1\|ld64\.so\.2" elif [[ "$ARCH" == aarch_64 ]]; then dump_cmd='objdump -p '"$1"' | grep NEEDED' - white_list="libpthread\.so\.0\|libc\.so\.6\|ld-linux-aarch64\.so\.1" + white_list="libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|ld-linux-aarch64\.so\.1" fi elif [[ "$OS" == osx ]]; then dump_cmd='otool -L '"$1"' | fgrep dylib' @@ -154,15 +165,10 @@ checkDependencies () } ############################################################################ -echo "Building protoc, OS=$OS ARCH=$ARCH TARGET=$TARGET" - -# Nested double quotes are unintuitive, but it works. -cd "$(dirname "$0")" +echo "Building protoc, OS=$OS ARCH=$ARCH TARGET=$MAKE_TARGET" -WORKING_DIR=$(pwd) CONFIGURE_ARGS="--disable-shared" -TARGET_FILE=target/$MAKE_TARGET.exe if [[ "$OS" == windows ]]; then MAKE_TARGET="${MAKE_TARGET}.exe" fi @@ -242,10 +248,18 @@ fi export CXXFLAGS LDFLAGS -cd "$WORKING_DIR"/.. && ./configure $CONFIGURE_ARGS && - cd src && make clean && make $MAKE_TARGET && - cd "$WORKING_DIR" && mkdir -p target && - cp ../src/$MAKE_TARGET $TARGET_FILE || +# Nested double quotes are unintuitive, but it works. +cd "$(dirname "$0")" + +WORKING_DIR="$(pwd)" +BUILD_DIR="build/$OS/$ARCH" +TARGET_FILE="target/$OS/$ARCH/$MAKE_TARGET.exe" + +mkdir -p "$BUILD_DIR" && cd "$BUILD_DIR" && + ../../../../configure $CONFIGURE_ARGS && + cd src && make $MAKE_TARGET -j8 && + cd "$WORKING_DIR" && mkdir -p $(dirname $TARGET_FILE) && + cp $BUILD_DIR/src/$MAKE_TARGET $TARGET_FILE || exit 1 if [[ "$OS" == osx ]]; then diff --git a/protoc-artifacts/pom.xml b/protoc-artifacts/pom.xml index 298730a6..390cd220 100644 --- a/protoc-artifacts/pom.xml +++ b/protoc-artifacts/pom.xml @@ -10,7 +10,7 @@ </parent> <groupId>com.google.protobuf</groupId> <artifactId>protoc</artifactId> - <version>3.5.2</version> + <version>3.6.0</version> <packaging>pom</packaging> <name>Protobuf Compiler</name> <description> @@ -33,38 +33,9 @@ </connection> </scm> <build> - <extensions> - <extension> - <groupId>kr.motd.maven</groupId> - <artifactId>os-maven-plugin</artifactId> - <version>1.5.0.Final</version> - </extension> - </extensions> <plugins> <plugin> <groupId>org.codehaus.mojo</groupId> - <artifactId>exec-maven-plugin</artifactId> - <version>1.1.1</version> - <executions> - <execution> - <phase>compile</phase> - <goals> - <goal>exec</goal> - </goals> - </execution> - </executions> - <configuration> - <executable>bash</executable> - <arguments> - <argument>build-protoc.sh</argument> - <argument>${os.detected.name}</argument> - <argument>${os.detected.arch}</argument> - <argument>protoc</argument> - </arguments> - </configuration> - </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> <artifactId>build-helper-maven-plugin</artifactId> <version>1.8</version> <executions> @@ -77,8 +48,38 @@ <configuration> <artifacts> <artifact> - <file>${basedir}/target/protoc.exe</file> - <classifier>${os.detected.name}-${os.detected.arch}</classifier> + <file>${basedir}/target/linux/x86_64/protoc.exe</file> + <classifier>linux-x86_64</classifier> + <type>exe</type> + </artifact> + <artifact> + <file>${basedir}/target/linux/x86_32/protoc.exe</file> + <classifier>linux-x86_32</classifier> + <type>exe</type> + </artifact> + <artifact> + <file>${basedir}/target/windows/x86_64/protoc.exe</file> + <classifier>windows-x86_64</classifier> + <type>exe</type> + </artifact> + <artifact> + <file>${basedir}/target/windows/x86_32/protoc.exe</file> + <classifier>windows-x86_32</classifier> + <type>exe</type> + </artifact> + <artifact> + <file>${basedir}/target/osx/x86_64/protoc.exe</file> + <classifier>osx-x86_64</classifier> + <type>exe</type> + </artifact> + <artifact> + <file>${basedir}/target/osx/x86_32/protoc.exe</file> + <classifier>osx-x86_32</classifier> + <type>exe</type> + </artifact> + <artifact> + <file>${basedir}/target/linux/aarch_64/protoc.exe</file> + <classifier>linux-aarch_64</classifier> <type>exe</type> </artifact> </artifacts> diff --git a/protoc-artifacts/scl-enable-devtoolset.sh b/protoc-artifacts/scl-enable-devtoolset.sh index 8d9585ea..30895585 100755 --- a/protoc-artifacts/scl-enable-devtoolset.sh +++ b/protoc-artifacts/scl-enable-devtoolset.sh @@ -10,4 +10,4 @@ quote() { done } -exec scl enable devtoolset-1.1 "$(quote "$@")" +exec scl enable devtoolset-2 "$(quote "$@")" diff --git a/python/google/protobuf/__init__.py b/python/google/protobuf/__init__.py index d4360727..58a3f040 100755 --- a/python/google/protobuf/__init__.py +++ b/python/google/protobuf/__init__.py @@ -30,7 +30,7 @@ # Copyright 2007 Google Inc. All Rights Reserved. -__version__ = '3.5.2' +__version__ = '3.6.0' if __name__ != '__main__': try: diff --git a/python/google/protobuf/proto_api.h b/python/google/protobuf/proto_api.h index 64d8dda9..5c076d23 100644 --- a/python/google/protobuf/proto_api.h +++ b/python/google/protobuf/proto_api.h @@ -81,7 +81,7 @@ struct PyProto_API { inline const char* PyProtoAPICapsuleName() { static const char kCapsuleName[] = - "protobuf.python.google.protobuf.cpp._message.proto_API"; + "google.protobuf.pyext._message.proto_API"; return kCapsuleName; } diff --git a/python/google/protobuf/pyext/descriptor.cc b/python/google/protobuf/pyext/descriptor.cc index 8af0cb12..19a1c38a 100644 --- a/python/google/protobuf/pyext/descriptor.cc +++ b/python/google/protobuf/pyext/descriptor.cc @@ -56,7 +56,7 @@ #endif #define PyString_AsStringAndSize(ob, charpp, sizep) \ (PyUnicode_Check(ob)? \ - ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \ + ((*(charpp) = const_cast<char*>(PyUnicode_AsUTF8AndSize(ob, (sizep)))) == NULL? -1: 0): \ PyBytes_AsStringAndSize(ob, (charpp), (sizep))) #endif diff --git a/python/google/protobuf/pyext/descriptor_containers.cc b/python/google/protobuf/pyext/descriptor_containers.cc index bc007f7e..0153664f 100644 --- a/python/google/protobuf/pyext/descriptor_containers.cc +++ b/python/google/protobuf/pyext/descriptor_containers.cc @@ -66,7 +66,7 @@ #endif #define PyString_AsStringAndSize(ob, charpp, sizep) \ (PyUnicode_Check(ob)? \ - ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \ + ((*(charpp) = const_cast<char*>(PyUnicode_AsUTF8AndSize(ob, (sizep)))) == NULL? -1: 0): \ PyBytes_AsStringAndSize(ob, (charpp), (sizep))) #endif diff --git a/python/google/protobuf/pyext/descriptor_pool.cc b/python/google/protobuf/pyext/descriptor_pool.cc index 95882aeb..962accc6 100644 --- a/python/google/protobuf/pyext/descriptor_pool.cc +++ b/python/google/protobuf/pyext/descriptor_pool.cc @@ -48,7 +48,7 @@ #endif #define PyString_AsStringAndSize(ob, charpp, sizep) \ (PyUnicode_Check(ob)? \ - ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \ + ((*(charpp) = const_cast<char*>(PyUnicode_AsUTF8AndSize(ob, (sizep)))) == NULL? -1: 0): \ PyBytes_AsStringAndSize(ob, (charpp), (sizep))) #endif diff --git a/python/google/protobuf/pyext/extension_dict.cc b/python/google/protobuf/pyext/extension_dict.cc index 018b5c2c..174c5470 100644 --- a/python/google/protobuf/pyext/extension_dict.cc +++ b/python/google/protobuf/pyext/extension_dict.cc @@ -53,7 +53,7 @@ #endif #define PyString_AsStringAndSize(ob, charpp, sizep) \ (PyUnicode_Check(ob)? \ - ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \ + ((*(charpp) = const_cast<char*>(PyUnicode_AsUTF8AndSize(ob, (sizep)))) == NULL? -1: 0): \ PyBytes_AsStringAndSize(ob, (charpp), (sizep))) #endif diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc index 53736b9c..b2984509 100644 --- a/python/google/protobuf/pyext/message.cc +++ b/python/google/protobuf/pyext/message.cc @@ -79,7 +79,7 @@ (PyUnicode_Check(ob)? PyUnicode_AsUTF8(ob): PyBytes_AsString(ob)) #define PyString_AsStringAndSize(ob, charpp, sizep) \ (PyUnicode_Check(ob)? \ - ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \ + ((*(charpp) = const_cast<char*>(PyUnicode_AsUTF8AndSize(ob, (sizep)))) == NULL? -1: 0): \ PyBytes_AsStringAndSize(ob, (charpp), (sizep))) #endif #endif @@ -1529,7 +1529,7 @@ PyObject* HasField(CMessage* self, PyObject* arg) { return NULL; } #else - field_name = PyUnicode_AsUTF8AndSize(arg, &size); + field_name = const_cast<char*>(PyUnicode_AsUTF8AndSize(arg, &size)); if (!field_name) { return NULL; } diff --git a/python/google/protobuf/pyext/message_module.cc b/python/google/protobuf/pyext/message_module.cc index f5c8f295..8c933866 100644 --- a/python/google/protobuf/pyext/message_module.cc +++ b/python/google/protobuf/pyext/message_module.cc @@ -130,6 +130,15 @@ extern "C" { Py_DECREF(m); return INITFUNC_ERRORVAL; } + + // Adds the C++ API + if (PyObject* api = + PyCapsule_New(new ApiImplementation(), + google::protobuf::python::PyProtoAPICapsuleName(), NULL)) { + PyModule_AddObject(m, "proto_API", api); + } else { + return INITFUNC_ERRORVAL; + } #if PY_MAJOR_VERSION >= 3 return m; diff --git a/python/setup.py b/python/setup.py index ebbe1bb6..335e9114 100755 --- a/python/setup.py +++ b/python/setup.py @@ -203,11 +203,9 @@ if __name__ == '__main__': v, _, _ = platform.mac_ver() if v: - v = float('.'.join(v.split('.')[:2])) - if v >= 10.12: - extra_compile_args.append('-std=c++11') + extra_compile_args.append('-std=c++11') elif os.getenv('KOKORO_BUILD_NUMBER') or os.getenv('KOKORO_BUILD_ID'): - extra_compile_args.append('-std=c++11') + extra_compile_args.append('-std=c++11') if warnings_as_errors in sys.argv: extra_compile_args.append('-Werror') diff --git a/python/tox.ini b/python/tox.ini index 38a81b4f..4eb319d2 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -4,7 +4,8 @@ envlist = [testenv] usedevelop=true -passenv = CC +passenv = + CC KOKORO_BUILD_ID KOKORO_BUILD_NUMBER setenv = cpp: LD_LIBRARY_PATH={toxinidir}/../src/.libs cpp: DYLD_LIBRARY_PATH={toxinidir}/../src/.libs diff --git a/ruby/compatibility_tests/v3.0.0/tests/basic.rb b/ruby/compatibility_tests/v3.0.0/tests/basic.rb index 05fe0278..e2559036 100644 --- a/ruby/compatibility_tests/v3.0.0/tests/basic.rb +++ b/ruby/compatibility_tests/v3.0.0/tests/basic.rb @@ -222,33 +222,55 @@ module BasicTest def test_type_errors m = TestMessage.new - assert_raise TypeError do + + # Use rescue to allow subclasses of error + success = false + begin m.optional_int32 = "hello" + rescue TypeError + success = true end - assert_raise TypeError do - m.optional_string = 42 - end - assert_raise TypeError do + assert(success) + + success = false + begin m.optional_string = nil + rescue TypeError + success = true end - assert_raise TypeError do + assert(success) + + success = false + begin m.optional_bool = 42 + rescue TypeError + success = true end - assert_raise TypeError do + assert(success) + + success = false + begin m.optional_msg = TestMessage.new # expects TestMessage2 + rescue TypeError + success = true end + assert(success) - assert_raise TypeError do + success = false + begin m.repeated_int32 = [] # needs RepeatedField + rescue TypeError + success = true end + assert(success) - assert_raise TypeError do - m.repeated_int32.push "hello" - end - - assert_raise TypeError do + success = false + begin m.repeated_msg.push TestMessage.new + rescue TypeError + success = true end + assert(success) end def test_string_encoding @@ -275,7 +297,7 @@ module BasicTest # strings are immutable so we can't do this, but serialize should catch it. m.optional_string = "asdf".encode!('UTF-8') - assert_raise RuntimeError do + assert_raise do m.optional_string.encode!('ASCII-8BIT') end end @@ -312,10 +334,14 @@ module BasicTest assert l.pop == 9 assert l == [5, 2, 3, 4, 7, 8] - assert_raise TypeError do + success = false + begin m = TestMessage.new l.push m + rescue TypeError + success = true end + assert(success) m = TestMessage.new m.repeated_int32 = l @@ -362,12 +388,22 @@ module BasicTest l = Google::Protobuf::RepeatedField.new(:message, TestMessage) l.push TestMessage.new assert l.count == 1 - assert_raise TypeError do + + success = false + begin l.push TestMessage2.new + rescue TypeError + success = true end - assert_raise TypeError do + assert(success) + + success = false + begin l.push 42 + rescue TypeError + success = true end + assert(success) l2 = l.dup assert l2[0] == l[0] @@ -493,9 +529,14 @@ module BasicTest assert m.length == 0 assert m == {} - assert_raise TypeError do + success = false + begin m[1] = 1 + rescue TypeError + success = true end + assert(success) + assert_raise RangeError do m["asdf"] = 0x1_0000_0000 end @@ -514,18 +555,28 @@ module BasicTest assert_raise RangeError do m[0x8000_0000] = 1 end - assert_raise TypeError do + + success = false + begin m["asdf"] = 1 + rescue TypeError + success = true end + assert(success) m = Google::Protobuf::Map.new(:int64, :int32) m[0x1000_0000_0000_0000] = 1 assert_raise RangeError do m[0x1_0000_0000_0000_0000] = 1 end - assert_raise TypeError do + + success = false + begin m["asdf"] = 1 + rescue TypeError + success = true end + assert(success) m = Google::Protobuf::Map.new(:uint32, :int32) m[0x8000_0000] = 1 @@ -548,18 +599,32 @@ module BasicTest m = Google::Protobuf::Map.new(:bool, :int32) m[true] = 1 m[false] = 2 - assert_raise TypeError do + + success = false + begin m[1] = 1 + rescue TypeError + success = true end - assert_raise TypeError do + assert(success) + + success = false + begin m["asdf"] = 1 + rescue TypeError + success = true end + assert(success) m = Google::Protobuf::Map.new(:string, :int32) m["asdf"] = 1 - assert_raise TypeError do + success = false + begin m[1] = 1 + rescue TypeError + success = true end + assert(success) assert_raise Encoding::UndefinedConversionError do bytestring = ["FFFF"].pack("H*") m[bytestring] = 1 @@ -570,17 +635,25 @@ module BasicTest m[bytestring] = 1 # Allowed -- we will automatically convert to ASCII-8BIT. m["asdf"] = 1 - assert_raise TypeError do + success = false + begin m[1] = 1 + rescue TypeError + success = true end + assert(success) end def test_map_msg_enum_valuetypes m = Google::Protobuf::Map.new(:string, :message, TestMessage) m["asdf"] = TestMessage.new - assert_raise TypeError do + success = false + begin m["jkl;"] = TestMessage2.new + rescue TypeError + success = true end + assert(success) m = Google::Protobuf::Map.new( :string, :message, TestMessage, @@ -645,23 +718,39 @@ module BasicTest m.map_string_msg.delete("c") assert m.map_string_msg == { "a" => TestMessage2.new(:foo => 1) } - assert_raise TypeError do + success = false + begin m.map_string_msg["e"] = TestMessage.new # wrong value type + rescue TypeError + success = true end + assert(success) # ensure nothing was added by the above assert m.map_string_msg == { "a" => TestMessage2.new(:foo => 1) } m.map_string_int32 = Google::Protobuf::Map.new(:string, :int32) - assert_raise TypeError do + success = false + begin m.map_string_int32 = Google::Protobuf::Map.new(:string, :int64) + rescue TypeError + success = true end - assert_raise TypeError do + assert(success) + success = false + begin m.map_string_int32 = {} + rescue TypeError + success = true end + assert(success) - assert_raise TypeError do + success = false + begin m = MapMessage.new(:map_string_int32 => { 1 => "I am not a number" }) + rescue TypeError + success = true end + assert(success) end def test_map_encode_decode @@ -922,22 +1011,30 @@ module BasicTest def test_def_errors s = Google::Protobuf::DescriptorPool.new - assert_raise TypeError do + success = false + begin s.build do # enum with no default (integer value 0) add_enum "MyEnum" do value :A, 1 end end + rescue TypeError + success = true end - assert_raise TypeError do + assert(success) + success = false + begin s.build do # message with required field (unsupported in proto3) add_message "MyMessage" do required :foo, :int32, 1 end end + rescue TypeError + success = true end + assert(success) end def test_corecursive diff --git a/ruby/compatibility_tests/v3.0.0/tests/repeated_field_test.rb b/ruby/compatibility_tests/v3.0.0/tests/repeated_field_test.rb index 25727b7b..201fe36b 100644 --- a/ruby/compatibility_tests/v3.0.0/tests/repeated_field_test.rb +++ b/ruby/compatibility_tests/v3.0.0/tests/repeated_field_test.rb @@ -18,7 +18,7 @@ class RepeatedFieldTest < Test::Unit::TestCase # jRuby additions to the Array class that we can ignore arr_methods -= [ :indices, :iter_for_each, :iter_for_each_index, :iter_for_each_with_index, :dimensions, :copy_data, :copy_data_simple, - :nitems, :iter_for_reverse_each, :indexes] + :nitems, :iter_for_reverse_each, :indexes, :append, :prepend] arr_methods.each do |method_name| assert m.repeated_string.respond_to?(method_name) == true, "does not respond to #{method_name}" end diff --git a/ruby/ext/google/protobuf_c/defs.c b/ruby/ext/google/protobuf_c/defs.c index d9d2ebac..9fe04503 100644 --- a/ruby/ext/google/protobuf_c/defs.c +++ b/ruby/ext/google/protobuf_c/defs.c @@ -812,7 +812,7 @@ VALUE FieldDescriptor_submsg_name_set(VALUE _self, VALUE value) { upb_fielddef* mut_def = check_field_notfrozen(self->fielddef); const char* str = get_str(value); if (!upb_fielddef_hassubdef(self->fielddef)) { - rb_raise(rb_eTypeError, "FieldDescriptor does not have subdef."); + rb_raise(cTypeError, "FieldDescriptor does not have subdef."); } CHECK_UPB(upb_fielddef_setsubdefname(mut_def, str, &status), "Error setting submessage name"); @@ -854,7 +854,7 @@ VALUE FieldDescriptor_get(VALUE _self, VALUE msg_rb) { MessageHeader* msg; TypedData_Get_Struct(msg_rb, MessageHeader, &Message_type, msg); if (msg->descriptor->msgdef != upb_fielddef_containingtype(self->fielddef)) { - rb_raise(rb_eTypeError, "get method called on wrong message type"); + rb_raise(cTypeError, "get method called on wrong message type"); } return layout_get(msg->descriptor->layout, Message_data(msg), self->fielddef); } @@ -872,7 +872,7 @@ VALUE FieldDescriptor_set(VALUE _self, VALUE msg_rb, VALUE value) { MessageHeader* msg; TypedData_Get_Struct(msg_rb, MessageHeader, &Message_type, msg); if (msg->descriptor->msgdef != upb_fielddef_containingtype(self->fielddef)) { - rb_raise(rb_eTypeError, "set method called on wrong message type"); + rb_raise(cTypeError, "set method called on wrong message type"); } layout_set(msg->descriptor->layout, Message_data(msg), self->fielddef, value); return Qnil; @@ -1713,7 +1713,7 @@ static void validate_msgdef(const upb_msgdef* msgdef) { upb_msg_field_next(&it)) { const upb_fielddef* field = upb_msg_iter_field(&it); if (upb_fielddef_label(field) == UPB_LABEL_REQUIRED) { - rb_raise(rb_eTypeError, "Required fields are unsupported in proto3."); + rb_raise(cTypeError, "Required fields are unsupported in proto3."); } } } @@ -1723,7 +1723,7 @@ static void validate_enumdef(const upb_enumdef* enumdef) { // value.) const char* lookup = upb_enumdef_iton(enumdef, 0); if (lookup == NULL) { - rb_raise(rb_eTypeError, + rb_raise(cTypeError, "Enum definition does not contain a value for '0'."); } } diff --git a/ruby/ext/google/protobuf_c/encode_decode.c b/ruby/ext/google/protobuf_c/encode_decode.c index 12080d03..c7a7aa1a 100644 --- a/ruby/ext/google/protobuf_c/encode_decode.c +++ b/ruby/ext/google/protobuf_c/encode_decode.c @@ -963,13 +963,15 @@ static void putary(VALUE ary, const upb_fielddef *f, upb_sink *sink, if (ary == Qnil) return; + size = NUM2INT(RepeatedField_length(ary)); + if (size == 0 && !emit_defaults) return; + upb_sink_startseq(sink, getsel(f, UPB_HANDLER_STARTSEQ), &subsink); if (upb_fielddef_isprimitive(f)) { sel = getsel(f, upb_handlers_getprimitivehandlertype(f)); } - size = NUM2INT(RepeatedField_length(ary)); for (int i = 0; i < size; i++) { void* memory = RepeatedField_index_native(ary, i); switch (type) { @@ -1106,6 +1108,13 @@ static void putmsg(VALUE msg_rb, const Descriptor* desc, TypedData_Get_Struct(msg_rb, MessageHeader, &Message_type, msg); + if (desc != msg->descriptor) { + rb_raise(cTypeError, + "The type of given msg is '%s', expect '%s'.", + upb_msgdef_fullname(msg->descriptor->msgdef), + upb_msgdef_fullname(desc->msgdef)); + } + for (upb_msg_field_begin(&i, desc->msgdef); !upb_msg_field_done(&i); upb_msg_field_next(&i)) { diff --git a/ruby/ext/google/protobuf_c/extconf.rb b/ruby/ext/google/protobuf_c/extconf.rb index 0886e607..cc097e11 100644 --- a/ruby/ext/google/protobuf_c/extconf.rb +++ b/ruby/ext/google/protobuf_c/extconf.rb @@ -2,7 +2,9 @@ require 'mkmf' -$CFLAGS += " -std=c99 -O3 -DNDEBUG" +unless RUBY_PLATFORM =~ /mswin|mingw/ + $CFLAGS += " -std=c99 -O3 -DNDEBUG" +end if RUBY_PLATFORM =~ /linux/ diff --git a/ruby/ext/google/protobuf_c/message.c b/ruby/ext/google/protobuf_c/message.c index 63a61baf..721c1112 100644 --- a/ruby/ext/google/protobuf_c/message.c +++ b/ruby/ext/google/protobuf_c/message.c @@ -256,6 +256,10 @@ int Message_initialize_kwarg(VALUE key, VALUE val, VALUE _self) { "Unknown field name '%s' in initialization map entry.", name); } + if (TYPE(val) == T_NIL) { + return 0; + } + if (is_map_field(f)) { VALUE map; @@ -631,7 +635,7 @@ VALUE build_module_from_enumdesc(EnumDescriptor* enumdesc) { const char* name = upb_enum_iter_name(&it); int32_t value = upb_enum_iter_number(&it); if (name[0] < 'A' || name[0] > 'Z') { - rb_raise(rb_eTypeError, + rb_raise(cTypeError, "Enum value '%s' does not start with an uppercase letter " "as is required for Ruby constants.", name); diff --git a/ruby/ext/google/protobuf_c/protobuf.c b/ruby/ext/google/protobuf_c/protobuf.c index db696426..fe6bb406 100644 --- a/ruby/ext/google/protobuf_c/protobuf.c +++ b/ruby/ext/google/protobuf_c/protobuf.c @@ -41,6 +41,7 @@ VALUE upb_def_to_ruby_obj_map; VALUE cError; VALUE cParseError; +VALUE cTypeError; void add_def_obj(const void* def, VALUE value) { rb_hash_aset(upb_def_to_ruby_obj_map, ULL2NUM((intptr_t)def), value); @@ -102,6 +103,7 @@ void Init_protobuf_c() { cError = rb_const_get(protobuf, rb_intern("Error")); cParseError = rb_const_get(protobuf, rb_intern("ParseError")); + cTypeError = rb_const_get(protobuf, rb_intern("TypeError")); rb_define_singleton_method(protobuf, "discard_unknown", Google_Protobuf_discard_unknown, 1); diff --git a/ruby/ext/google/protobuf_c/protobuf.h b/ruby/ext/google/protobuf_c/protobuf.h index 5266aa8d..3e5c0520 100644 --- a/ruby/ext/google/protobuf_c/protobuf.h +++ b/ruby/ext/google/protobuf_c/protobuf.h @@ -161,6 +161,7 @@ extern VALUE cBuilder; extern VALUE cError; extern VALUE cParseError; +extern VALUE cTypeError; // We forward-declare all of the Ruby method implementations here because we // sometimes call the methods directly across .c files, rather than going diff --git a/ruby/ext/google/protobuf_c/storage.c b/ruby/ext/google/protobuf_c/storage.c index 1437c0b5..163b2f81 100644 --- a/ruby/ext/google/protobuf_c/storage.c +++ b/ruby/ext/google/protobuf_c/storage.c @@ -96,7 +96,7 @@ static bool is_ruby_num(VALUE value) { void native_slot_check_int_range_precision(upb_fieldtype_t type, VALUE val) { if (!is_ruby_num(val)) { - rb_raise(rb_eTypeError, "Expected number type for integral field."); + rb_raise(cTypeError, "Expected number type for integral field."); } // NUM2{INT,UINT,LL,ULL} macros do the appropriate range checks on upper @@ -153,13 +153,13 @@ void native_slot_set_value_and_case(upb_fieldtype_t type, VALUE type_class, switch (type) { case UPB_TYPE_FLOAT: if (!is_ruby_num(value)) { - rb_raise(rb_eTypeError, "Expected number type for float field."); + rb_raise(cTypeError, "Expected number type for float field."); } DEREF(memory, float) = NUM2DBL(value); break; case UPB_TYPE_DOUBLE: if (!is_ruby_num(value)) { - rb_raise(rb_eTypeError, "Expected number type for double field."); + rb_raise(cTypeError, "Expected number type for double field."); } DEREF(memory, double) = NUM2DBL(value); break; @@ -170,16 +170,16 @@ void native_slot_set_value_and_case(upb_fieldtype_t type, VALUE type_class, } else if (value == Qfalse) { val = 0; } else { - rb_raise(rb_eTypeError, "Invalid argument for boolean field."); + rb_raise(cTypeError, "Invalid argument for boolean field."); } DEREF(memory, int8_t) = val; break; } case UPB_TYPE_STRING: if (CLASS_OF(value) == rb_cSymbol) { - value = rb_funcall(value, rb_intern("to_s"), 0, NULL); + value = rb_funcall(value, rb_intern("to_s"), 0); } else if (CLASS_OF(value) != rb_cString) { - rb_raise(rb_eTypeError, "Invalid argument for string field."); + rb_raise(cTypeError, "Invalid argument for string field."); } DEREF(memory, VALUE) = native_slot_encode_and_freeze_string(type, value); @@ -187,7 +187,7 @@ void native_slot_set_value_and_case(upb_fieldtype_t type, VALUE type_class, case UPB_TYPE_BYTES: { if (CLASS_OF(value) != rb_cString) { - rb_raise(rb_eTypeError, "Invalid argument for string field."); + rb_raise(cTypeError, "Invalid argument for string field."); } DEREF(memory, VALUE) = native_slot_encode_and_freeze_string(type, value); @@ -197,7 +197,7 @@ void native_slot_set_value_and_case(upb_fieldtype_t type, VALUE type_class, if (CLASS_OF(value) == CLASS_OF(Qnil)) { value = Qnil; } else if (CLASS_OF(value) != type_class) { - rb_raise(rb_eTypeError, + rb_raise(cTypeError, "Invalid type %s to assign to submessage field.", rb_class2name(CLASS_OF(value))); } @@ -207,9 +207,9 @@ void native_slot_set_value_and_case(upb_fieldtype_t type, VALUE type_class, case UPB_TYPE_ENUM: { int32_t int_val = 0; if (TYPE(value) == T_STRING) { - value = rb_funcall(value, rb_intern("to_sym"), 0, NULL); + value = rb_funcall(value, rb_intern("to_sym"), 0); } else if (!is_ruby_num(value) && TYPE(value) != T_SYMBOL) { - rb_raise(rb_eTypeError, + rb_raise(cTypeError, "Expected number or symbol type for enum field."); } if (TYPE(value) == T_SYMBOL) { @@ -598,18 +598,18 @@ static void check_repeated_field_type(VALUE val, const upb_fielddef* field) { if (!RB_TYPE_P(val, T_DATA) || !RTYPEDDATA_P(val) || RTYPEDDATA_TYPE(val) != &RepeatedField_type) { - rb_raise(rb_eTypeError, "Expected repeated field array"); + rb_raise(cTypeError, "Expected repeated field array"); } self = ruby_to_RepeatedField(val); if (self->field_type != upb_fielddef_type(field)) { - rb_raise(rb_eTypeError, "Repeated field array has wrong element type"); + rb_raise(cTypeError, "Repeated field array has wrong element type"); } - if (self->field_type == UPB_TYPE_MESSAGE) { + if (self->field_type == UPB_TYPE_MESSAGE) { if (self->field_type_class != Descriptor_msgclass(get_def_obj(upb_fielddef_subdef(field)))) { - rb_raise(rb_eTypeError, + rb_raise(cTypeError, "Repeated field array has wrong message class"); } } @@ -618,7 +618,7 @@ static void check_repeated_field_type(VALUE val, const upb_fielddef* field) { if (self->field_type == UPB_TYPE_ENUM) { if (self->field_type_class != EnumDescriptor_enummodule(get_def_obj(upb_fielddef_subdef(field)))) { - rb_raise(rb_eTypeError, + rb_raise(cTypeError, "Repeated field array has wrong enum class"); } } @@ -631,21 +631,21 @@ static void check_map_field_type(VALUE val, const upb_fielddef* field) { if (!RB_TYPE_P(val, T_DATA) || !RTYPEDDATA_P(val) || RTYPEDDATA_TYPE(val) != &Map_type) { - rb_raise(rb_eTypeError, "Expected Map instance"); + rb_raise(cTypeError, "Expected Map instance"); } self = ruby_to_Map(val); if (self->key_type != upb_fielddef_type(key_field)) { - rb_raise(rb_eTypeError, "Map key type does not match field's key type"); + rb_raise(cTypeError, "Map key type does not match field's key type"); } if (self->value_type != upb_fielddef_type(value_field)) { - rb_raise(rb_eTypeError, "Map value type does not match field's value type"); + rb_raise(cTypeError, "Map value type does not match field's value type"); } if (upb_fielddef_type(value_field) == UPB_TYPE_MESSAGE || upb_fielddef_type(value_field) == UPB_TYPE_ENUM) { if (self->value_type_class != get_def_obj(upb_fielddef_subdef(value_field))) { - rb_raise(rb_eTypeError, + rb_raise(cTypeError, "Map value type has wrong message/enum class"); } } diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index c09a7f24..cc7625d4 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = "google-protobuf" - s.version = "3.5.2" + s.version = "3.6.0" s.licenses = ["BSD-3-Clause"] s.summary = "Protocol Buffers" s.description = "Protocol Buffers are Google's data interchange format." diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb index 4a805e88..e20a584e 100644 --- a/ruby/lib/google/protobuf.rb +++ b/ruby/lib/google/protobuf.rb @@ -37,6 +37,7 @@ module Google module Protobuf class Error < StandardError; end class ParseError < Error; end + class TypeError < ::TypeError; end end end diff --git a/ruby/lib/google/protobuf/well_known_types.rb b/ruby/lib/google/protobuf/well_known_types.rb index 3e759591..2ee65bc2 100644 --- a/ruby/lib/google/protobuf/well_known_types.rb +++ b/ruby/lib/google/protobuf/well_known_types.rb @@ -155,6 +155,8 @@ module Google Struct.class_eval do def [](key) self.fields[key].to_ruby + rescue NoMethodError + nil end def []=(key, value) @@ -176,6 +178,10 @@ module Google hash.each { |key, val| ret[key] = val } ret end + + def has_key?(key) + self.fields.has_key?(key) + end end ListValue.class_eval do diff --git a/ruby/src/main/java/com/google/protobuf/jruby/RubyMessage.java b/ruby/src/main/java/com/google/protobuf/jruby/RubyMessage.java index 07558fbc..c3a0d81c 100644 --- a/ruby/src/main/java/com/google/protobuf/jruby/RubyMessage.java +++ b/ruby/src/main/java/com/google/protobuf/jruby/RubyMessage.java @@ -86,6 +86,8 @@ public class RubyMessage extends RubyObject { throw runtime.newTypeError("Expected string or symbols as hash keys in initialization map."); final Descriptors.FieldDescriptor fieldDescriptor = findField(context, key); + if (value.isNil()) return; + if (Utils.isMapEntry(fieldDescriptor)) { if (!(value instanceof RubyHash)) throw runtime.newArgumentError("Expected Hash object as initializer value for map field '" + key.asJavaString() + "'."); diff --git a/ruby/tests/basic.rb b/ruby/tests/basic.rb index ad34d53d..f174402b 100644 --- a/ruby/tests/basic.rb +++ b/ruby/tests/basic.rb @@ -212,6 +212,15 @@ module BasicTest assert_equal ['foo', 'bar'], m.repeated_string end + def test_ctor_nil_args + m = TestMessage.new(:optional_enum => nil, :optional_int32 => nil, :optional_string => nil, :optional_msg => nil) + + assert_equal :Default, m.optional_enum + assert_equal 0, m.optional_int32 + assert_equal "", m.optional_string + assert_nil m.optional_msg + end + def test_embeddedmsg_hash_init m = TestEmbeddedMessageParent.new(:child_msg => {sub_child: {optional_int32: 1}}, :number => 2, @@ -283,31 +292,36 @@ module BasicTest def test_type_errors m = TestMessage.new - assert_raise TypeError do + e = assert_raise Google::Protobuf::TypeError do m.optional_int32 = "hello" end - assert_raise TypeError do + + # Google::Protobuf::TypeError should inherit from TypeError for backwards compatibility + # TODO: This can be removed when we can safely migrate to Google::Protobuf::TypeError + assert_true e.is_a?(::TypeError) + + assert_raise Google::Protobuf::TypeError do m.optional_string = 42 end - assert_raise TypeError do + assert_raise Google::Protobuf::TypeError do m.optional_string = nil end - assert_raise TypeError do + assert_raise Google::Protobuf::TypeError do m.optional_bool = 42 end - assert_raise TypeError do + assert_raise Google::Protobuf::TypeError do m.optional_msg = TestMessage.new # expects TestMessage2 end - assert_raise TypeError do + assert_raise Google::Protobuf::TypeError do m.repeated_int32 = [] # needs RepeatedField end - assert_raise TypeError do + assert_raise Google::Protobuf::TypeError do m.repeated_int32.push "hello" end - assert_raise TypeError do + assert_raise Google::Protobuf::TypeError do m.repeated_msg.push TestMessage.new end end @@ -336,7 +350,9 @@ module BasicTest # strings are immutable so we can't do this, but serialize should catch it. m.optional_string = "asdf".encode!('UTF-8') - assert_raise RuntimeError do + # Ruby 2.5 changed to raise FrozenError. However, assert_raise don't + # accept subclass. Don't specify type here. + assert_raise do m.optional_string.encode!('ASCII-8BIT') end end @@ -373,7 +389,7 @@ module BasicTest assert l.pop == 9 assert l == [5, 2, 3, 4, 7, 8] - assert_raise TypeError do + assert_raise Google::Protobuf::TypeError do m = TestMessage.new l.push m end @@ -423,10 +439,10 @@ module BasicTest l = Google::Protobuf::RepeatedField.new(:message, TestMessage) l.push TestMessage.new assert l.count == 1 - assert_raise TypeError do + assert_raise Google::Protobuf::TypeError do l.push TestMessage2.new end - assert_raise TypeError do + assert_raise Google::Protobuf::TypeError do l.push 42 end @@ -575,7 +591,7 @@ module BasicTest assert_raise RangeError do m[0x8000_0000] = 1 end - assert_raise TypeError do + assert_raise Google::Protobuf::TypeError do m["asdf"] = 1 end @@ -584,7 +600,7 @@ module BasicTest assert_raise RangeError do m[0x1_0000_0000_0000_0000] = 1 end - assert_raise TypeError do + assert_raise Google::Protobuf::TypeError do m["asdf"] = 1 end @@ -609,10 +625,10 @@ module BasicTest m = Google::Protobuf::Map.new(:bool, :int32) m[true] = 1 m[false] = 2 - assert_raise TypeError do + assert_raise Google::Protobuf::TypeError do m[1] = 1 end - assert_raise TypeError do + assert_raise Google::Protobuf::TypeError do m["asdf"] = 1 end @@ -639,7 +655,7 @@ module BasicTest def test_map_msg_enum_valuetypes m = Google::Protobuf::Map.new(:string, :message, TestMessage) m["asdf"] = TestMessage.new - assert_raise TypeError do + assert_raise Google::Protobuf::TypeError do m["jkl;"] = TestMessage2.new end @@ -706,17 +722,17 @@ module BasicTest m.map_string_msg.delete("c") assert m.map_string_msg == { "a" => TestMessage2.new(:foo => 1) } - assert_raise TypeError do + assert_raise Google::Protobuf::TypeError do m.map_string_msg["e"] = TestMessage.new # wrong value type end # ensure nothing was added by the above assert m.map_string_msg == { "a" => TestMessage2.new(:foo => 1) } m.map_string_int32 = Google::Protobuf::Map.new(:string, :int32) - assert_raise TypeError do + assert_raise Google::Protobuf::TypeError do m.map_string_int32 = Google::Protobuf::Map.new(:string, :int64) end - assert_raise TypeError do + assert_raise Google::Protobuf::TypeError do m.map_string_int32 = {} end @@ -1015,7 +1031,7 @@ module BasicTest def test_def_errors s = Google::Protobuf::DescriptorPool.new - assert_raise TypeError do + assert_raise Google::Protobuf::TypeError do s.build do # enum with no default (integer value 0) add_enum "MyEnum" do @@ -1023,7 +1039,7 @@ module BasicTest end end end - assert_raise TypeError do + assert_raise Google::Protobuf::TypeError do s.build do # message with required field (unsupported in proto3) add_message "MyMessage" do @@ -1259,6 +1275,10 @@ module BasicTest Foo.encode_json(Foo.new(bar: bar, baz: [baz1, baz2])) end + def test_json_empty + assert TestMessage.encode_json(TestMessage.new) == '{}' + end + def test_json_emit_defaults # TODO: Fix JSON in JRuby version. return if RUBY_PLATFORM == "java" diff --git a/ruby/tests/encode_decode_test.rb b/ruby/tests/encode_decode_test.rb index 2bd9b98b..95c2cfa4 100644 --- a/ruby/tests/encode_decode_test.rb +++ b/ruby/tests/encode_decode_test.rb @@ -84,4 +84,14 @@ class EncodeDecodeTest < Test::Unit::TestCase assert_match 'optional_int32', json end + + def test_encode_wrong_msg + e = assert_raise Google::Protobuf::TypeError do + m = A::B::C::TestMessage.new( + :optional_int32 => 1, + ) + Google::Protobuf::Any.encode(m) + end + end + end diff --git a/ruby/tests/repeated_field_test.rb b/ruby/tests/repeated_field_test.rb index 61ac4afd..a4f3aab8 100644 --- a/ruby/tests/repeated_field_test.rb +++ b/ruby/tests/repeated_field_test.rb @@ -18,7 +18,7 @@ class RepeatedFieldTest < Test::Unit::TestCase # jRuby additions to the Array class that we can ignore arr_methods -= [ :indices, :iter_for_each, :iter_for_each_index, :iter_for_each_with_index, :dimensions, :copy_data, :copy_data_simple, - :nitems, :iter_for_reverse_each, :indexes] + :nitems, :iter_for_reverse_each, :indexes, :append, :prepend] arr_methods.each do |method_name| assert m.repeated_string.respond_to?(method_name) == true, "does not respond to #{method_name}" end diff --git a/ruby/tests/well_known_types_test.rb b/ruby/tests/well_known_types_test.rb index 240281e7..f35f7b13 100644 --- a/ruby/tests/well_known_types_test.rb +++ b/ruby/tests/well_known_types_test.rb @@ -60,6 +60,9 @@ class TestWellKnownTypes < Test::Unit::TestCase assert_equal(Google::Protobuf::ListValue.from_a(sublist), struct["sublist"]) + assert_equal true, struct.has_key?("null") + assert_equal false, struct.has_key?("missing_key") + should_equal = { "number" => 12345, "boolean-true" => true, @@ -82,6 +85,9 @@ class TestWellKnownTypes < Test::Unit::TestCase # to_h returns a fully-flattened Ruby structure (Hash and Array). assert_equal(should_equal, struct.to_h) + # Test that we can safely access a missing key + assert_equal(nil, struct["missing_key"]) + # Test that we can assign Struct and ListValue directly. struct["substruct"] = Google::Protobuf::Struct.from_hash(substruct) struct["sublist"] = Google::Protobuf::ListValue.from_a(sublist) diff --git a/src/Makefile.am b/src/Makefile.am index b8648049..cb1b1006 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -164,7 +164,7 @@ nobase_include_HEADERS = \ lib_LTLIBRARIES = libprotobuf-lite.la libprotobuf.la libprotoc.la libprotobuf_lite_la_LIBADD = $(PTHREAD_LIBS) -libprotobuf_lite_la_LDFLAGS = -version-info 15:1:0 -export-dynamic -no-undefined +libprotobuf_lite_la_LDFLAGS = -version-info 16:0:0 -export-dynamic -no-undefined if HAVE_LD_VERSION_SCRIPT libprotobuf_lite_la_LDFLAGS += -Wl,--version-script=$(srcdir)/libprotobuf-lite.map EXTRA_libprotobuf_lite_la_DEPENDENCIES = libprotobuf-lite.map @@ -210,7 +210,7 @@ libprotobuf_lite_la_SOURCES = \ google/protobuf/io/zero_copy_stream_impl_lite.cc libprotobuf_la_LIBADD = $(PTHREAD_LIBS) -libprotobuf_la_LDFLAGS = -version-info 15:1:0 -export-dynamic -no-undefined +libprotobuf_la_LDFLAGS = -version-info 16:0:0 -export-dynamic -no-undefined if HAVE_LD_VERSION_SCRIPT libprotobuf_la_LDFLAGS += -Wl,--version-script=$(srcdir)/libprotobuf.map EXTRA_libprotobuf_la_DEPENDENCIES = libprotobuf.map @@ -301,7 +301,7 @@ libprotobuf_la_SOURCES = \ nodist_libprotobuf_la_SOURCES = $(nodist_libprotobuf_lite_la_SOURCES) libprotoc_la_LIBADD = $(PTHREAD_LIBS) libprotobuf.la -libprotoc_la_LDFLAGS = -version-info 15:1:0 -export-dynamic -no-undefined +libprotoc_la_LDFLAGS = -version-info 16:0:0 -export-dynamic -no-undefined if HAVE_LD_VERSION_SCRIPT libprotoc_la_LDFLAGS += -Wl,--version-script=$(srcdir)/libprotoc.map EXTRA_libprotoc_la_DEPENDENCIES = libprotoc.map @@ -876,7 +876,7 @@ no_warning_test.cc: no_warning_test_LDADD = $(PTHREAD_LIBS) libprotobuf.la libprotoc.la no_warning_test_CXXFLAGS = $(PTHREAD_CFLAGS) $(PTHREAD_DEF) $(ZLIB_DEF) \ - -Wall -Wextra -Werror -Wno-unused-parameter + -Wall -Wextra -Werror -Wno-unused-parameter -Og nodist_no_warning_test_SOURCES = no_warning_test.cc $(protoc_outputs) TESTS = protobuf-test protobuf-lazy-descriptor-test protobuf-lite-test \ diff --git a/src/google/protobuf/any.pb.h b/src/google/protobuf/any.pb.h index bc537999..8a4c7f97 100644 --- a/src/google/protobuf/any.pb.h +++ b/src/google/protobuf/any.pb.h @@ -8,12 +8,12 @@ #include <google/protobuf/stubs/common.h> -#if GOOGLE_PROTOBUF_VERSION < 3005000 +#if GOOGLE_PROTOBUF_VERSION < 3006000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3005001 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION +#if 3006000 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/api.pb.cc b/src/google/protobuf/api.pb.cc index 6ad9f48d..273307c7 100644 --- a/src/google/protobuf/api.pb.cc +++ b/src/google/protobuf/api.pb.cc @@ -414,7 +414,7 @@ bool Api::MergePartialFromCodedStream( case 7: { if (static_cast< ::google::protobuf::uint8>(tag) == static_cast< ::google::protobuf::uint8>(56u /* 56 & 0xFF */)) { - int value; + int value = 0; DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( input, &value))); @@ -947,7 +947,7 @@ bool Method::MergePartialFromCodedStream( case 7: { if (static_cast< ::google::protobuf::uint8>(tag) == static_cast< ::google::protobuf::uint8>(56u /* 56 & 0xFF */)) { - int value; + int value = 0; DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( input, &value))); diff --git a/src/google/protobuf/api.pb.h b/src/google/protobuf/api.pb.h index 5a720cce..0f90f411 100644 --- a/src/google/protobuf/api.pb.h +++ b/src/google/protobuf/api.pb.h @@ -8,12 +8,12 @@ #include <google/protobuf/stubs/common.h> -#if GOOGLE_PROTOBUF_VERSION < 3005000 +#if GOOGLE_PROTOBUF_VERSION < 3006000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3005001 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION +#if 3006000 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/compiler/cpp/cpp_enum_field.cc b/src/google/protobuf/compiler/cpp/cpp_enum_field.cc index 50c7b5f3..828d0be5 100644 --- a/src/google/protobuf/compiler/cpp/cpp_enum_field.cc +++ b/src/google/protobuf/compiler/cpp/cpp_enum_field.cc @@ -128,7 +128,7 @@ GenerateCopyConstructorCode(io::Printer* printer) const { void EnumFieldGenerator:: GenerateMergeFromCodedStream(io::Printer* printer) const { printer->Print(variables_, - "int value;\n" + "int value = 0;\n" "DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive<\n" " int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>(\n" " input, &value)));\n"); @@ -334,7 +334,7 @@ void RepeatedEnumFieldGenerator:: GenerateMergeFromCodedStream(io::Printer* printer) const { // Don't use ReadRepeatedPrimitive here so that the enum can be validated. printer->Print(variables_, - "int value;\n" + "int value = 0;\n" "DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive<\n" " int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>(\n" " input, &value)));\n"); @@ -399,7 +399,7 @@ GenerateMergeFromCodedStreamWithPacking(io::Printer* printer) const { "::google::protobuf::io::CodedInputStream::Limit limit = " "input->PushLimit(static_cast<int>(length));\n" "while (input->BytesUntilLimit() > 0) {\n" - " int value;\n" + " int value = 0;\n" " DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive<\n" " int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>(\n" " input, &value)));\n"); diff --git a/src/google/protobuf/compiler/js/js_generator.cc b/src/google/protobuf/compiler/js/js_generator.cc index 7109ed5b..45b906f5 100644 --- a/src/google/protobuf/compiler/js/js_generator.cc +++ b/src/google/protobuf/compiler/js/js_generator.cc @@ -276,7 +276,8 @@ string GetEnumPath(const GeneratorOptions& options, string MaybeCrossFileRef(const GeneratorOptions& options, const FileDescriptor* from_file, const Descriptor* to_message) { - if (options.import_style == GeneratorOptions::kImportCommonJs && + if ((options.import_style == GeneratorOptions::kImportCommonJs || + options.import_style == GeneratorOptions::kImportCommonJsStrict) && from_file != to_message->file()) { // Cross-file ref in CommonJS needs to use the module alias instead of // the global name. @@ -1675,8 +1676,19 @@ void Generator::GenerateProvides(const GeneratorOptions& options, // // // Later generated code expects foo.bar = {} to exist: // foo.bar.Baz = function() { /* ... */ } - printer->Print("goog.exportSymbol('$name$', null, global);\n", "name", - *it); + + // Do not use global scope in strict mode + if (options.import_style == GeneratorOptions::kImportCommonJsStrict) { + string namespaceObject = *it; + // Remove "proto." from the namespace object + GOOGLE_CHECK(namespaceObject.compare(0, 6, "proto.") == 0); + namespaceObject.erase(0, 6); + printer->Print("goog.exportSymbol('$name$', null, proto);\n", "name", + namespaceObject); + } else { + printer->Print("goog.exportSymbol('$name$', null, global);\n", "name", + *it); + } } } } @@ -2962,8 +2974,12 @@ void Generator::GenerateClassDeserializeBinaryField( if (value_field->type() == FieldDescriptor::TYPE_MESSAGE) { printer->Print(", $messageType$.deserializeBinaryFromReader", "messageType", GetMessagePath(options, value_field->message_type())); + } else { + printer->Print(", null"); } - + printer->Print(", $defaultKey$", + "defaultKey", JSFieldDefault(key_field) + ); printer->Print(");\n"); printer->Print(" });\n"); } else { @@ -3321,6 +3337,8 @@ bool GeneratorOptions::ParseFromOptions( import_style = kImportClosure; } else if (options[i].second == "commonjs") { import_style = kImportCommonJs; + } else if (options[i].second == "commonjs_strict") { + import_style = kImportCommonJsStrict; } else if (options[i].second == "browser") { import_style = kImportBrowser; } else if (options[i].second == "es6") { @@ -3430,15 +3448,23 @@ void Generator::GenerateFile(const GeneratorOptions& options, GenerateHeader(options, printer); // Generate "require" statements. - if (options.import_style == GeneratorOptions::kImportCommonJs) { + if ((options.import_style == GeneratorOptions::kImportCommonJs || + options.import_style == GeneratorOptions::kImportCommonJsStrict)) { printer->Print("var jspb = require('google-protobuf');\n"); printer->Print("var goog = jspb;\n"); - printer->Print("var global = Function('return this')();\n\n"); + + // Do not use global scope in strict mode + if (options.import_style == GeneratorOptions::kImportCommonJsStrict) { + printer->Print("var proto = {};\n\n"); + } else { + printer->Print("var global = Function('return this')();\n\n"); + } for (int i = 0; i < file->dependency_count(); i++) { const string& name = file->dependency(i)->name(); printer->Print( - "var $alias$ = require('$file$');\n", + "var $alias$ = require('$file$');\n" + "goog.object.extend(proto, $alias$);\n", "alias", ModuleAlias(name), "file", GetRootPath(file->name(), name) + GetJSFilename(options, name)); @@ -3481,6 +3507,9 @@ void Generator::GenerateFile(const GeneratorOptions& options, if (options.import_style == GeneratorOptions::kImportCommonJs && !provided.empty()) { printer->Print("goog.object.extend(exports, $package$);\n", "package", GetFilePath(options, file)); + } else if(options.import_style == GeneratorOptions::kImportCommonJsStrict) { + printer->Print("goog.object.extend(exports, proto);\n", + "package", GetFilePath(options, file)); } // Emit well-known type methods. diff --git a/src/google/protobuf/compiler/js/js_generator.h b/src/google/protobuf/compiler/js/js_generator.h index 3cc60e22..b50ef7fd 100644 --- a/src/google/protobuf/compiler/js/js_generator.h +++ b/src/google/protobuf/compiler/js/js_generator.h @@ -63,10 +63,11 @@ struct GeneratorOptions { bool binary; // What style of imports should be used. enum ImportStyle { - kImportClosure, // goog.require() - kImportCommonJs, // require() - kImportBrowser, // no import statements - kImportEs6, // import { member } from '' + kImportClosure, // goog.require() + kImportCommonJs, // require() + kImportCommonJsStrict, // require() with no global export + kImportBrowser, // no import statements + kImportEs6, // import { member } from '' } import_style; GeneratorOptions() diff --git a/src/google/protobuf/compiler/plugin.pb.h b/src/google/protobuf/compiler/plugin.pb.h index 5ad6b4d3..d5b70a88 100644 --- a/src/google/protobuf/compiler/plugin.pb.h +++ b/src/google/protobuf/compiler/plugin.pb.h @@ -8,12 +8,12 @@ #include <google/protobuf/stubs/common.h> -#if GOOGLE_PROTOBUF_VERSION < 3005000 +#if GOOGLE_PROTOBUF_VERSION < 3006000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3005001 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION +#if 3006000 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index 695a7b02..fd003994 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -479,15 +479,8 @@ typedef std::map<DescriptorIntPair, const FieldDescriptor*> ExtensionsGroupedByDescriptorMap; typedef HASH_MAP<string, const SourceCodeInfo_Location*> LocationsByPathMap; -std::set<string>* allowed_proto3_extendees_ = NULL; -GOOGLE_PROTOBUF_DECLARE_ONCE(allowed_proto3_extendees_init_); - -void DeleteAllowedProto3Extendee() { - delete allowed_proto3_extendees_; -} - -void InitAllowedProto3Extendee() { - allowed_proto3_extendees_ = new std::set<string>; +std::set<string>* NewAllowedProto3Extendee() { + auto allowed_proto3_extendees = new std::set<string>; const char* kOptionNames[] = { "FileOptions", "MessageOptions", "FieldOptions", "EnumOptions", "EnumValueOptions", "ServiceOptions", "MethodOptions", "OneofOptions"}; @@ -495,14 +488,13 @@ void InitAllowedProto3Extendee() { // descriptor.proto has a different package name in opensource. We allow // both so the opensource protocol compiler can also compile internal // proto3 files with custom options. See: b/27567912 - allowed_proto3_extendees_->insert(string("google.protobuf.") + + allowed_proto3_extendees->insert(string("google.protobuf.") + kOptionNames[i]); // Split the word to trick the opensource processing scripts so they // will keep the origial package name. - allowed_proto3_extendees_->insert(string("proto") + "2." + kOptionNames[i]); + allowed_proto3_extendees->insert(string("proto") + "2." + kOptionNames[i]); } - - google::protobuf::internal::OnShutdown(&DeleteAllowedProto3Extendee); + return allowed_proto3_extendees; } // Checks whether the extendee type is allowed in proto3. @@ -510,9 +502,10 @@ void InitAllowedProto3Extendee() { // instead of comparing the descriptor directly because the extensions may be // defined in a different pool. bool AllowedExtendeeInProto3(const string& name) { - ::google::protobuf::GoogleOnceInit(&allowed_proto3_extendees_init_, &InitAllowedProto3Extendee); - return allowed_proto3_extendees_->find(name) != - allowed_proto3_extendees_->end(); + static auto allowed_proto3_extendees = + internal::OnShutdownDelete(NewAllowedProto3Extendee()); + return allowed_proto3_extendees->find(name) != + allowed_proto3_extendees->end(); } } // anonymous namespace @@ -775,10 +768,10 @@ class FileDescriptorTables { SymbolsByParentMap symbols_by_parent_; mutable FieldsByNameMap fields_by_lowercase_name_; - mutable FieldsByNameMap* fields_by_lowercase_name_tmp_; + std::unique_ptr<FieldsByNameMap> fields_by_lowercase_name_tmp_; mutable GoogleOnceDynamic fields_by_lowercase_name_once_; mutable FieldsByNameMap fields_by_camelcase_name_; - mutable FieldsByNameMap* fields_by_camelcase_name_tmp_; + std::unique_ptr<FieldsByNameMap> fields_by_camelcase_name_tmp_; mutable GoogleOnceDynamic fields_by_camelcase_name_once_; FieldsByNumberMap fields_by_number_; // Not including extensions. EnumValuesByNumberMap enum_values_by_number_; @@ -829,31 +822,10 @@ FileDescriptorTables::FileDescriptorTables() FileDescriptorTables::~FileDescriptorTables() {} -namespace { - -FileDescriptorTables* file_descriptor_tables_ = NULL; -GOOGLE_PROTOBUF_DECLARE_ONCE(file_descriptor_tables_once_init_); - -void DeleteFileDescriptorTables() { - delete file_descriptor_tables_; - file_descriptor_tables_ = NULL; -} - -void InitFileDescriptorTables() { - file_descriptor_tables_ = new FileDescriptorTables(); - internal::OnShutdown(&DeleteFileDescriptorTables); -} - -inline void InitFileDescriptorTablesOnce() { - ::google::protobuf::GoogleOnceInit( - &file_descriptor_tables_once_init_, &InitFileDescriptorTables); -} - -} // anonymous namespace - inline const FileDescriptorTables& FileDescriptorTables::GetEmptyInstance() { - InitFileDescriptorTablesOnce(); - return *file_descriptor_tables_; + static auto file_descriptor_tables = + internal::OnShutdownDelete(new FileDescriptorTables()); + return *file_descriptor_tables; } void DescriptorPool::Tables::AddCheckpoint() { @@ -1146,10 +1118,8 @@ bool DescriptorPool::Tables::AddFile(const FileDescriptor* file) { void FileDescriptorTables::FinalizeTables() { // Clean up the temporary maps used by AddFieldByStylizedNames(). - delete fields_by_lowercase_name_tmp_; - fields_by_lowercase_name_tmp_ = NULL; - delete fields_by_camelcase_name_tmp_; - fields_by_camelcase_name_tmp_ = NULL; + fields_by_lowercase_name_tmp_ = nullptr; + fields_by_camelcase_name_tmp_ = nullptr; } void FileDescriptorTables::AddFieldByStylizedNames( @@ -1164,7 +1134,7 @@ void FileDescriptorTables::AddFieldByStylizedNames( // entries from fields_by_number_. PointerStringPair lowercase_key(parent, field->lowercase_name().c_str()); - if (!InsertIfNotPresent(fields_by_lowercase_name_tmp_, lowercase_key, + if (!InsertIfNotPresent(fields_by_lowercase_name_tmp_.get(), lowercase_key, field)) { InsertIfNotPresent( &fields_by_lowercase_name_, lowercase_key, @@ -1172,7 +1142,7 @@ void FileDescriptorTables::AddFieldByStylizedNames( } PointerStringPair camelcase_key(parent, field->camelcase_name().c_str()); - if (!InsertIfNotPresent(fields_by_camelcase_name_tmp_, camelcase_key, + if (!InsertIfNotPresent(fields_by_camelcase_name_tmp_.get(), camelcase_key, field)) { InsertIfNotPresent( &fields_by_camelcase_name_, camelcase_key, @@ -1337,42 +1307,28 @@ bool DescriptorPool::InternalIsFileLoaded(const string& filename) const { namespace { - -EncodedDescriptorDatabase* generated_database_ = NULL; -DescriptorPool* generated_pool_ = NULL; -GOOGLE_PROTOBUF_DECLARE_ONCE(generated_pool_init_); - -void DeleteGeneratedPool() { - delete generated_database_; - generated_database_ = NULL; - delete generated_pool_; - generated_pool_ = NULL; -} - -static void InitGeneratedPool() { - generated_database_ = new EncodedDescriptorDatabase; - generated_pool_ = new DescriptorPool(generated_database_); - generated_pool_->InternalSetLazilyBuildDependencies(); - - internal::OnShutdown(&DeleteGeneratedPool); +EncodedDescriptorDatabase* GeneratedDatabase() { + static auto generated_database = + internal::OnShutdownDelete(new EncodedDescriptorDatabase()); + return generated_database; } -inline void InitGeneratedPoolOnce() { - ::google::protobuf::GoogleOnceInit(&generated_pool_init_, &InitGeneratedPool); +DescriptorPool* NewGeneratedPool() { + auto generated_pool = new DescriptorPool(GeneratedDatabase()); + generated_pool->InternalSetLazilyBuildDependencies(); + return generated_pool; } } // anonymous namespace -const DescriptorPool* DescriptorPool::generated_pool() { - InitGeneratedPoolOnce(); - return generated_pool_; +DescriptorPool* DescriptorPool::internal_generated_pool() { + static DescriptorPool* generated_pool = + internal::OnShutdownDelete(NewGeneratedPool()); + return generated_pool; } - - -DescriptorPool* DescriptorPool::internal_generated_pool() { - InitGeneratedPoolOnce(); - return generated_pool_; +const DescriptorPool* DescriptorPool::generated_pool() { + return internal_generated_pool(); } void DescriptorPool::InternalAddGeneratedFile( @@ -1399,8 +1355,7 @@ void DescriptorPool::InternalAddGeneratedFile( // Therefore, when we parse one, we have to be very careful to avoid using // any descriptor-based operations, since this might cause infinite recursion // or deadlock. - InitGeneratedPoolOnce(); - GOOGLE_CHECK(generated_database_->Add(encoded_file_descriptor, size)); + GOOGLE_CHECK(GeneratedDatabase()->Add(encoded_file_descriptor, size)); } diff --git a/src/google/protobuf/descriptor.pb.cc b/src/google/protobuf/descriptor.pb.cc index 2ccf189c..f3522ec1 100644 --- a/src/google/protobuf/descriptor.pb.cc +++ b/src/google/protobuf/descriptor.pb.cc @@ -4507,7 +4507,7 @@ bool FieldDescriptorProto::MergePartialFromCodedStream( case 4: { if (static_cast< ::google::protobuf::uint8>(tag) == static_cast< ::google::protobuf::uint8>(32u /* 32 & 0xFF */)) { - int value; + int value = 0; DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( input, &value))); @@ -4527,7 +4527,7 @@ bool FieldDescriptorProto::MergePartialFromCodedStream( case 5: { if (static_cast< ::google::protobuf::uint8>(tag) == static_cast< ::google::protobuf::uint8>(40u /* 40 & 0xFF */)) { - int value; + int value = 0; DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( input, &value))); @@ -7814,7 +7814,7 @@ bool FileOptions::MergePartialFromCodedStream( case 9: { if (static_cast< ::google::protobuf::uint8>(tag) == static_cast< ::google::protobuf::uint8>(72u /* 72 & 0xFF */)) { - int value; + int value = 0; DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( input, &value))); @@ -9377,7 +9377,7 @@ bool FieldOptions::MergePartialFromCodedStream( case 1: { if (static_cast< ::google::protobuf::uint8>(tag) == static_cast< ::google::protobuf::uint8>(8u /* 8 & 0xFF */)) { - int value; + int value = 0; DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( input, &value))); @@ -9439,7 +9439,7 @@ bool FieldOptions::MergePartialFromCodedStream( case 6: { if (static_cast< ::google::protobuf::uint8>(tag) == static_cast< ::google::protobuf::uint8>(48u /* 48 & 0xFF */)) { - int value; + int value = 0; DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( input, &value))); @@ -11237,7 +11237,7 @@ bool MethodOptions::MergePartialFromCodedStream( case 34: { if (static_cast< ::google::protobuf::uint8>(tag) == static_cast< ::google::protobuf::uint8>(16u /* 272 & 0xFF */)) { - int value; + int value = 0; DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( input, &value))); diff --git a/src/google/protobuf/descriptor.pb.h b/src/google/protobuf/descriptor.pb.h index 5e7a05d5..7a8617a2 100644 --- a/src/google/protobuf/descriptor.pb.h +++ b/src/google/protobuf/descriptor.pb.h @@ -8,12 +8,12 @@ #include <google/protobuf/stubs/common.h> -#if GOOGLE_PROTOBUF_VERSION < 3005000 +#if GOOGLE_PROTOBUF_VERSION < 3006000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3005001 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION +#if 3006000 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/duration.pb.cc b/src/google/protobuf/duration.pb.cc index 0c67f86e..e8417a78 100644 --- a/src/google/protobuf/duration.pb.cc +++ b/src/google/protobuf/duration.pb.cc @@ -167,7 +167,7 @@ void Duration::ArenaDtor(void* object) { Duration* _this = reinterpret_cast< Duration* >(object); (void)_this; } -void Duration::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void Duration::RegisterArenaDtor(::google::protobuf::Arena*) { } void Duration::SetCachedSize(int size) const { _cached_size_.Set(size); diff --git a/src/google/protobuf/duration.pb.h b/src/google/protobuf/duration.pb.h index 3d0889d1..751edbef 100644 --- a/src/google/protobuf/duration.pb.h +++ b/src/google/protobuf/duration.pb.h @@ -8,12 +8,12 @@ #include <google/protobuf/stubs/common.h> -#if GOOGLE_PROTOBUF_VERSION < 3005000 +#if GOOGLE_PROTOBUF_VERSION < 3006000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3005001 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION +#if 3006000 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/empty.pb.cc b/src/google/protobuf/empty.pb.cc index 7bc46121..163a4f6a 100644 --- a/src/google/protobuf/empty.pb.cc +++ b/src/google/protobuf/empty.pb.cc @@ -156,7 +156,7 @@ void Empty::ArenaDtor(void* object) { Empty* _this = reinterpret_cast< Empty* >(object); (void)_this; } -void Empty::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void Empty::RegisterArenaDtor(::google::protobuf::Arena*) { } void Empty::SetCachedSize(int size) const { _cached_size_.Set(size); diff --git a/src/google/protobuf/empty.pb.h b/src/google/protobuf/empty.pb.h index 53857e5f..c3da4fa8 100644 --- a/src/google/protobuf/empty.pb.h +++ b/src/google/protobuf/empty.pb.h @@ -8,12 +8,12 @@ #include <google/protobuf/stubs/common.h> -#if GOOGLE_PROTOBUF_VERSION < 3005000 +#if GOOGLE_PROTOBUF_VERSION < 3006000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3005001 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION +#if 3006000 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/extension_set.cc b/src/google/protobuf/extension_set.cc index b587b38f..cb205c4f 100644 --- a/src/google/protobuf/extension_set.cc +++ b/src/google/protobuf/extension_set.cc @@ -36,7 +36,6 @@ #include <tuple> #include <utility> #include <google/protobuf/stubs/common.h> -#include <google/protobuf/stubs/once.h> #include <google/protobuf/extension_set.h> #include <google/protobuf/message_lite.h> #include <google/protobuf/io/coded_stream.h> @@ -80,27 +79,17 @@ inline bool is_packable(WireFormatLite::WireType type) { // Registry stuff. typedef hash_map<std::pair<const MessageLite*, int>, ExtensionInfo> ExtensionRegistry; -ExtensionRegistry* registry_ = NULL; -GOOGLE_PROTOBUF_DECLARE_ONCE(registry_init_); -void DeleteRegistry() { - delete registry_; - registry_ = NULL; -} - -void InitRegistry() { - registry_ = new ExtensionRegistry; - OnShutdown(&DeleteRegistry); -} +static const ExtensionRegistry* global_registry = nullptr; // This function is only called at startup, so there is no need for thread- // safety. void Register(const MessageLite* containing_type, int number, ExtensionInfo info) { - ::google::protobuf::GoogleOnceInit(®istry_init_, &InitRegistry); - - if (!InsertIfNotPresent(registry_, std::make_pair(containing_type, number), - info)) { + static auto local_static_registry = OnShutdownDelete(new ExtensionRegistry); + global_registry = local_static_registry; + if (!InsertIfNotPresent(local_static_registry, + std::make_pair(containing_type, number), info)) { GOOGLE_LOG(FATAL) << "Multiple extension registrations for type \"" << containing_type->GetTypeName() << "\", field number " << number << "."; @@ -109,9 +98,9 @@ void Register(const MessageLite* containing_type, const ExtensionInfo* FindRegisteredExtension( const MessageLite* containing_type, int number) { - return (registry_ == NULL) - ? NULL - : FindOrNull(*registry_, std::make_pair(containing_type, number)); + return global_registry == nullptr + ? nullptr + : FindOrNull(*global_registry, std::make_pair(containing_type, number)); } } // namespace @@ -1911,67 +1900,16 @@ void ExtensionSet::Erase(int key) { // ================================================================== // Default repeated field instances for iterator-compatible accessors -GOOGLE_PROTOBUF_DECLARE_ONCE(repeated_primitive_generic_type_traits_once_init_); -GOOGLE_PROTOBUF_DECLARE_ONCE(repeated_string_type_traits_once_init_); -GOOGLE_PROTOBUF_DECLARE_ONCE(repeated_message_generic_type_traits_once_init_); - -void RepeatedPrimitiveGenericTypeTraits::InitializeDefaultRepeatedFields() { - default_repeated_field_int32_ = new RepeatedField<int32>; - default_repeated_field_int64_ = new RepeatedField<int64>; - default_repeated_field_uint32_ = new RepeatedField<uint32>; - default_repeated_field_uint64_ = new RepeatedField<uint64>; - default_repeated_field_double_ = new RepeatedField<double>; - default_repeated_field_float_ = new RepeatedField<float>; - default_repeated_field_bool_ = new RepeatedField<bool>; - OnShutdown(&DestroyDefaultRepeatedFields); -} - -void RepeatedPrimitiveGenericTypeTraits::DestroyDefaultRepeatedFields() { - delete default_repeated_field_int32_; - delete default_repeated_field_int64_; - delete default_repeated_field_uint32_; - delete default_repeated_field_uint64_; - delete default_repeated_field_double_; - delete default_repeated_field_float_; - delete default_repeated_field_bool_; -} - -void RepeatedStringTypeTraits::InitializeDefaultRepeatedFields() { - default_repeated_field_ = new RepeatedFieldType; - OnShutdown(&DestroyDefaultRepeatedFields); -} - -void RepeatedStringTypeTraits::DestroyDefaultRepeatedFields() { - delete default_repeated_field_; -} - -void RepeatedMessageGenericTypeTraits::InitializeDefaultRepeatedFields() { - default_repeated_field_ = new RepeatedFieldType; - OnShutdown(&DestroyDefaultRepeatedFields); -} - -void RepeatedMessageGenericTypeTraits::DestroyDefaultRepeatedFields() { - delete default_repeated_field_; +const RepeatedPrimitiveDefaults* RepeatedPrimitiveDefaults::default_instance() { + static auto instance = OnShutdownDelete(new RepeatedPrimitiveDefaults); + return instance; } -const RepeatedField<int32>* -RepeatedPrimitiveGenericTypeTraits::default_repeated_field_int32_ = NULL; -const RepeatedField<int64>* -RepeatedPrimitiveGenericTypeTraits::default_repeated_field_int64_ = NULL; -const RepeatedField<uint32>* -RepeatedPrimitiveGenericTypeTraits::default_repeated_field_uint32_ = NULL; -const RepeatedField<uint64>* -RepeatedPrimitiveGenericTypeTraits::default_repeated_field_uint64_ = NULL; -const RepeatedField<double>* -RepeatedPrimitiveGenericTypeTraits::default_repeated_field_double_ = NULL; -const RepeatedField<float>* -RepeatedPrimitiveGenericTypeTraits::default_repeated_field_float_ = NULL; -const RepeatedField<bool>* -RepeatedPrimitiveGenericTypeTraits::default_repeated_field_bool_ = NULL; const RepeatedStringTypeTraits::RepeatedFieldType* -RepeatedStringTypeTraits::default_repeated_field_ = NULL; -const RepeatedMessageGenericTypeTraits::RepeatedFieldType* -RepeatedMessageGenericTypeTraits::default_repeated_field_ = NULL; +RepeatedStringTypeTraits::GetDefaultRepeatedField() { + static auto instance = OnShutdownDelete(new RepeatedFieldType); + return instance; +} } // namespace internal } // namespace protobuf diff --git a/src/google/protobuf/extension_set.h b/src/google/protobuf/extension_set.h index e94e0b9c..04d0e62c 100644 --- a/src/google/protobuf/extension_set.h +++ b/src/google/protobuf/extension_set.h @@ -880,18 +880,17 @@ class RepeatedPrimitiveTypeTraits { LIBPROTOBUF_EXPORT extern ProtobufOnceType repeated_primitive_generic_type_traits_once_init_; -class LIBPROTOBUF_EXPORT RepeatedPrimitiveGenericTypeTraits { +class LIBPROTOBUF_EXPORT RepeatedPrimitiveDefaults { private: template<typename Type> friend class RepeatedPrimitiveTypeTraits; - static void InitializeDefaultRepeatedFields(); - static void DestroyDefaultRepeatedFields(); - static const RepeatedField<int32>* default_repeated_field_int32_; - static const RepeatedField<int64>* default_repeated_field_int64_; - static const RepeatedField<uint32>* default_repeated_field_uint32_; - static const RepeatedField<uint64>* default_repeated_field_uint64_; - static const RepeatedField<double>* default_repeated_field_double_; - static const RepeatedField<float>* default_repeated_field_float_; - static const RepeatedField<bool>* default_repeated_field_bool_; + static const RepeatedPrimitiveDefaults* default_instance(); + RepeatedField<int32> default_repeated_field_int32_; + RepeatedField<int64> default_repeated_field_int64_; + RepeatedField<uint32> default_repeated_field_uint32_; + RepeatedField<uint64> default_repeated_field_uint64_; + RepeatedField<double> default_repeated_field_double_; + RepeatedField<float> default_repeated_field_float_; + RepeatedField<bool> default_repeated_field_bool_; }; #define PROTOBUF_DEFINE_PRIMITIVE_TYPE(TYPE, METHOD) \ @@ -919,11 +918,8 @@ template<> inline void RepeatedPrimitiveTypeTraits<TYPE>::Add( \ } \ template<> inline const RepeatedField<TYPE>* \ RepeatedPrimitiveTypeTraits<TYPE>::GetDefaultRepeatedField() { \ - ::google::protobuf::GoogleOnceInit( \ - &repeated_primitive_generic_type_traits_once_init_, \ - &RepeatedPrimitiveGenericTypeTraits::InitializeDefaultRepeatedFields); \ - return RepeatedPrimitiveGenericTypeTraits:: \ - default_repeated_field_##TYPE##_; \ + return &RepeatedPrimitiveDefaults::default_instance() \ + ->default_repeated_field_##TYPE##_; \ } \ template<> inline const RepeatedField<TYPE>& \ RepeatedPrimitiveTypeTraits<TYPE>::GetRepeated(int number, \ @@ -980,8 +976,6 @@ class LIBPROTOBUF_EXPORT StringTypeTraits { } }; -LIBPROTOBUF_EXPORT extern ProtobufOnceType repeated_string_type_traits_once_init_; - class LIBPROTOBUF_EXPORT RepeatedStringTypeTraits { public: typedef const string& ConstType; @@ -1024,11 +1018,7 @@ class LIBPROTOBUF_EXPORT RepeatedStringTypeTraits { is_packed, NULL)); } - static const RepeatedFieldType* GetDefaultRepeatedField() { - ::google::protobuf::GoogleOnceInit(&repeated_string_type_traits_once_init_, - &InitializeDefaultRepeatedFields); - return default_repeated_field_; - } + static const RepeatedFieldType* GetDefaultRepeatedField(); template <typename ExtendeeT> static void Register(int number, FieldType type, bool is_packed) { @@ -1039,7 +1029,6 @@ class LIBPROTOBUF_EXPORT RepeatedStringTypeTraits { private: static void InitializeDefaultRepeatedFields(); static void DestroyDefaultRepeatedFields(); - static const RepeatedFieldType *default_repeated_field_; }; // ------------------------------------------------------------------- @@ -1230,28 +1219,11 @@ class RepeatedMessageTypeTraits { } }; -LIBPROTOBUF_EXPORT extern ProtobufOnceType repeated_message_generic_type_traits_once_init_; - -// This class exists only to hold a generic default empty repeated field for all -// message-type repeated field extensions. -class LIBPROTOBUF_EXPORT RepeatedMessageGenericTypeTraits { - public: - typedef RepeatedPtrField<::google::protobuf::MessageLite*> RepeatedFieldType; - private: - template<typename Type> friend class RepeatedMessageTypeTraits; - static void InitializeDefaultRepeatedFields(); - static void DestroyDefaultRepeatedFields(); - static const RepeatedFieldType* default_repeated_field_; -}; - template<typename Type> inline const typename RepeatedMessageTypeTraits<Type>::RepeatedFieldType* RepeatedMessageTypeTraits<Type>::GetDefaultRepeatedField() { - ::google::protobuf::GoogleOnceInit( - &repeated_message_generic_type_traits_once_init_, - &RepeatedMessageGenericTypeTraits::InitializeDefaultRepeatedFields); - return reinterpret_cast<const RepeatedFieldType*>( - RepeatedMessageGenericTypeTraits::default_repeated_field_); + static auto instance = OnShutdownDelete(new RepeatedFieldType); + return instance; } // ------------------------------------------------------------------- diff --git a/src/google/protobuf/field_mask.pb.h b/src/google/protobuf/field_mask.pb.h index 144b41fa..7550f7a9 100644 --- a/src/google/protobuf/field_mask.pb.h +++ b/src/google/protobuf/field_mask.pb.h @@ -8,12 +8,12 @@ #include <google/protobuf/stubs/common.h> -#if GOOGLE_PROTOBUF_VERSION < 3005000 +#if GOOGLE_PROTOBUF_VERSION < 3006000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3005001 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION +#if 3006000 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/generated_message_reflection.cc b/src/google/protobuf/generated_message_reflection.cc index 247f772c..74ad00e7 100644 --- a/src/google/protobuf/generated_message_reflection.cc +++ b/src/google/protobuf/generated_message_reflection.cc @@ -2327,32 +2327,26 @@ class AssignDescriptorsHelper { // automatically delete the allocated reflection. MetadataOwner owns // all the allocated reflection instances. struct MetadataOwner { + ~MetadataOwner() { + for (auto range : metadata_arrays_) { + for (const Metadata* m = range.first; m < range.second; m++) { + delete m->reflection; + } + } + } + void AddArray(const Metadata* begin, const Metadata* end) { MutexLock lock(&mu_); metadata_arrays_.push_back(std::make_pair(begin, end)); } static MetadataOwner* Instance() { - static MetadataOwner* res = new MetadataOwner; + static MetadataOwner* res = OnShutdownDelete(new MetadataOwner); return res; } private: - // Use the constructor to register the shutdown code. Because c++ makes sure - // this called only once. - MetadataOwner() { OnShutdown(&DeleteMetadata); } - ~MetadataOwner() { - for (int i = 0; i < metadata_arrays_.size(); i++) { - for (const Metadata* m = metadata_arrays_[i].first; - m < metadata_arrays_[i].second; m++) { - delete m->reflection; - } - } - } - - static void DeleteMetadata() { - delete Instance(); - } + MetadataOwner() = default; // private because singleton Mutex mu_; std::vector<std::pair<const Metadata*, const Metadata*> > metadata_arrays_; diff --git a/src/google/protobuf/generated_message_util.cc b/src/google/protobuf/generated_message_util.cc index dac8ca90..e0241361 100644 --- a/src/google/protobuf/generated_message_util.cc +++ b/src/google/protobuf/generated_message_util.cc @@ -57,6 +57,12 @@ namespace google { namespace protobuf { namespace internal { +void DestroyMessage(const void* message) { + static_cast<const MessageLite*>(message)->~MessageLite(); +} +void DestroyString(const void* s) { static_cast<const string*>(s)->~string(); } + +ExplicitlyConstructed<std::string> fixed_address_empty_string; double Infinity() { return std::numeric_limits<double>::infinity(); @@ -65,14 +71,15 @@ double NaN() { return std::numeric_limits<double>::quiet_NaN(); } -ExplicitlyConstructed<::std::string> fixed_address_empty_string; -GOOGLE_PROTOBUF_DECLARE_ONCE(empty_string_once_init_); - -void DeleteEmptyString() { fixed_address_empty_string.Destruct(); } - -void InitEmptyString() { +static bool InitProtobufDefaultsImpl() { fixed_address_empty_string.DefaultConstruct(); - OnShutdown(&DeleteEmptyString); + OnShutdownDestroyString(fixed_address_empty_string.get_mutable()); + return true; +} + +void InitProtobufDefaults() { + static bool is_inited = InitProtobufDefaultsImpl(); + (void)is_inited; } size_t StringSpaceUsedExcludingSelfLong(const string& str) { @@ -86,12 +93,6 @@ size_t StringSpaceUsedExcludingSelfLong(const string& str) { } } - - -void InitProtobufDefaults() { - GetEmptyString(); -} - template <typename T> const T& Get(const void* ptr) { return *static_cast<const T*>(ptr); diff --git a/src/google/protobuf/generated_message_util.h b/src/google/protobuf/generated_message_util.h index bdaa561b..76b81242 100644 --- a/src/google/protobuf/generated_message_util.h +++ b/src/google/protobuf/generated_message_util.h @@ -109,6 +109,13 @@ namespace internal { LIBPROTOBUF_EXPORT double Infinity(); LIBPROTOBUF_EXPORT double NaN(); +LIBPROTOBUF_EXPORT void InitProtobufDefaults(); + +// This used by proto1 +inline const std::string& GetEmptyString() { + InitProtobufDefaults(); + return GetEmptyStringAlreadyInited(); +} // True if IsInitialized() is true for all elements of t. Type is expected // to be a RepeatedPtrField<some message type>. It's useful to have this @@ -137,8 +144,6 @@ bool AllAreInitializedWeak(const ::google::protobuf::RepeatedPtrField<T>& t) { return true; } -LIBPROTOBUF_EXPORT void InitProtobufDefaults(); - struct LIBPROTOBUF_EXPORT FieldMetadata { uint32 offset; // offset of this field in the struct uint32 tag; // field * 8 + wire_type @@ -335,7 +340,16 @@ struct LIBPROTOBUF_EXPORT SCCInfoBase { kRunning = 1, kUninitialized = -1, // initial state }; +#ifndef _MSC_VER std::atomic<int> visit_status; +#else + // MSVC doesnt make std::atomic constant initialized. This union trick + // makes it so. + union { + int visit_status_to_make_linker_init; + std::atomic<int> visit_status; + }; +#endif int num_deps; void (*init_func)(); // This is followed by an array of num_deps @@ -359,6 +373,17 @@ inline void InitSCC(SCCInfoBase* scc) { if (GOOGLE_PREDICT_FALSE(status != SCCInfoBase::kInitialized)) InitSCCImpl(scc); } +LIBPROTOBUF_EXPORT void DestroyMessage(const void* message); +LIBPROTOBUF_EXPORT void DestroyString(const void* s); +// Destroy (not delete) the message +inline void OnShutdownDestroyMessage(const void* ptr) { + OnShutdownRun(DestroyMessage, ptr); +} +// Destroy the string (call string destructor) +inline void OnShutdownDestroyString(const std::string* ptr) { + OnShutdownRun(DestroyString, ptr); +} + } // namespace internal } // namespace protobuf diff --git a/src/google/protobuf/io/gzip_stream.h b/src/google/protobuf/io/gzip_stream.h index 15b02fe3..df1a446e 100644 --- a/src/google/protobuf/io/gzip_stream.h +++ b/src/google/protobuf/io/gzip_stream.h @@ -118,7 +118,7 @@ class LIBPROTOBUF_EXPORT GzipOutputStream : public ZeroCopyOutputStream { ZLIB = 2, }; - struct Options { + struct LIBPROTOBUF_EXPORT Options { // Defaults to GZIP. Format format; diff --git a/src/google/protobuf/map_entry.h b/src/google/protobuf/map_entry.h index f25b5834..f170f97b 100644 --- a/src/google/protobuf/map_entry.h +++ b/src/google/protobuf/map_entry.h @@ -95,12 +95,12 @@ class MapEntry typedef void InternalArenaConstructable_; typedef void DestructorSkippable_; - using typename MapEntryImpl<Derived, Message, Key, Value, kKeyFieldType, - kValueFieldType, - default_enum_value>::KeyTypeHandler; - using typename MapEntryImpl<Derived, Message, Key, Value, kKeyFieldType, - kValueFieldType, - default_enum_value>::ValueTypeHandler; + typedef typename MapEntryImpl< + Derived, Message, Key, Value, kKeyFieldType, kValueFieldType, + default_enum_value>::KeyTypeHandler KeyTypeHandler; + typedef typename MapEntryImpl< + Derived, Message, Key, Value, kKeyFieldType, kValueFieldType, + default_enum_value>::ValueTypeHandler ValueTypeHandler; size_t SpaceUsedLong() const override { size_t size = sizeof(Derived); size += KeyTypeHandler::SpaceUsedInMapEntryLong(this->key_); diff --git a/src/google/protobuf/message.cc b/src/google/protobuf/message.cc index 7fda0c79..a536615a 100644 --- a/src/google/protobuf/message.cc +++ b/src/google/protobuf/message.cc @@ -262,9 +262,6 @@ namespace { class GeneratedMessageFactory : public MessageFactory { public: - GeneratedMessageFactory(); - ~GeneratedMessageFactory(); - static GeneratedMessageFactory* singleton(); typedef void RegistrationFunc(const string&); @@ -284,25 +281,9 @@ class GeneratedMessageFactory : public MessageFactory { hash_map<const Descriptor*, const Message*> type_map_; }; -GeneratedMessageFactory* generated_message_factory_ = NULL; -GOOGLE_PROTOBUF_DECLARE_ONCE(generated_message_factory_once_init_); - -void ShutdownGeneratedMessageFactory() { - delete generated_message_factory_; -} - -void InitGeneratedMessageFactory() { - generated_message_factory_ = new GeneratedMessageFactory; - internal::OnShutdown(&ShutdownGeneratedMessageFactory); -} - -GeneratedMessageFactory::GeneratedMessageFactory() {} -GeneratedMessageFactory::~GeneratedMessageFactory() {} - GeneratedMessageFactory* GeneratedMessageFactory::singleton() { - ::google::protobuf::GoogleOnceInit(&generated_message_factory_once_init_, - &InitGeneratedMessageFactory); - return generated_message_factory_; + static auto instance = internal::OnShutdownDelete(new GeneratedMessageFactory); + return instance; } void GeneratedMessageFactory::RegisterFile( diff --git a/src/google/protobuf/message_lite.h b/src/google/protobuf/message_lite.h index 4cbec330..b8644142 100644 --- a/src/google/protobuf/message_lite.h +++ b/src/google/protobuf/message_lite.h @@ -129,19 +129,11 @@ class ExplicitlyConstructed { // Default empty string object. Don't use this directly. Instead, call // GetEmptyString() to get the reference. LIBPROTOBUF_EXPORT extern ExplicitlyConstructed<::std::string> fixed_address_empty_string; -LIBPROTOBUF_EXPORT extern ProtobufOnceType empty_string_once_init_; -LIBPROTOBUF_EXPORT void InitEmptyString(); - LIBPROTOBUF_EXPORT inline const ::std::string& GetEmptyStringAlreadyInited() { return fixed_address_empty_string.get(); } -LIBPROTOBUF_EXPORT inline const ::std::string& GetEmptyString() { - ::google::protobuf::GoogleOnceInit(&empty_string_once_init_, &InitEmptyString); - return GetEmptyStringAlreadyInited(); -} - LIBPROTOBUF_EXPORT size_t StringSpaceUsedExcludingSelfLong(const string& str); #endif // SWIG } // namespace internal diff --git a/src/google/protobuf/repeated_field.h b/src/google/protobuf/repeated_field.h index b47ea994..79682b69 100644 --- a/src/google/protobuf/repeated_field.h +++ b/src/google/protobuf/repeated_field.h @@ -466,7 +466,7 @@ class LIBPROTOBUF_EXPORT RepeatedPtrFieldBase { protected: template <typename TypeHandler> void Add(typename TypeHandler::Type&& value, - std::enable_if<TypeHandler::Moveable>* dummy = NULL); + typename std::enable_if<TypeHandler::Moveable>::type* dummy = NULL); template <typename TypeHandler> void RemoveLast(); @@ -705,13 +705,13 @@ void GenericTypeHandler<string>::Merge(const string& from, // Declarations of the specialization as we cannot define them here, as the // header that defines ProtocolMessage depends on types defined in this header. #define DECLARE_SPECIALIZATIONS_FOR_BASE_PROTO_TYPES(TypeName) \ - template<> \ + template<> LIBPROTOBUF_EXPORT \ TypeName* GenericTypeHandler<TypeName>::NewFromPrototype( \ const TypeName* prototype, google::protobuf::Arena* arena); \ - template<> \ + template<> LIBPROTOBUF_EXPORT \ google::protobuf::Arena* GenericTypeHandler<TypeName>::GetArena( \ TypeName* value); \ - template<> \ + template<> LIBPROTOBUF_EXPORT \ void* GenericTypeHandler<TypeName>::GetMaybeArenaPointer( \ TypeName* value); @@ -1541,7 +1541,7 @@ inline typename TypeHandler::Type* RepeatedPtrFieldBase::Add( template <typename TypeHandler> inline void RepeatedPtrFieldBase::Add( typename TypeHandler::Type&& value, - std::enable_if<TypeHandler::Moveable>*) { + typename std::enable_if<TypeHandler::Moveable>::type*) { if (rep_ != NULL && current_size_ < rep_->allocated_size) { *cast<TypeHandler>(rep_->elements[current_size_++]) = std::move(value); return; diff --git a/src/google/protobuf/source_context.pb.h b/src/google/protobuf/source_context.pb.h index 16ed21ca..eee1b73f 100644 --- a/src/google/protobuf/source_context.pb.h +++ b/src/google/protobuf/source_context.pb.h @@ -8,12 +8,12 @@ #include <google/protobuf/stubs/common.h> -#if GOOGLE_PROTOBUF_VERSION < 3005000 +#if GOOGLE_PROTOBUF_VERSION < 3006000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3005001 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION +#if 3006000 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/struct.pb.cc b/src/google/protobuf/struct.pb.cc index 4bf5d45c..ead81689 100644 --- a/src/google/protobuf/struct.pb.cc +++ b/src/google/protobuf/struct.pb.cc @@ -277,7 +277,7 @@ void Struct::ArenaDtor(void* object) { Struct* _this = reinterpret_cast< Struct* >(object); (void)_this; } -void Struct::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void Struct::RegisterArenaDtor(::google::protobuf::Arena*) { } void Struct::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -730,7 +730,7 @@ void Value::ArenaDtor(void* object) { Value* _this = reinterpret_cast< Value* >(object); (void)_this; } -void Value::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void Value::RegisterArenaDtor(::google::protobuf::Arena*) { } void Value::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -810,7 +810,7 @@ bool Value::MergePartialFromCodedStream( case 1: { if (static_cast< ::google::protobuf::uint8>(tag) == static_cast< ::google::protobuf::uint8>(8u /* 8 & 0xFF */)) { - int value; + int value = 0; DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( input, &value))); @@ -1225,7 +1225,7 @@ void ListValue::ArenaDtor(void* object) { ListValue* _this = reinterpret_cast< ListValue* >(object); (void)_this; } -void ListValue::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void ListValue::RegisterArenaDtor(::google::protobuf::Arena*) { } void ListValue::SetCachedSize(int size) const { _cached_size_.Set(size); diff --git a/src/google/protobuf/struct.pb.h b/src/google/protobuf/struct.pb.h index 75665ff2..b2263435 100644 --- a/src/google/protobuf/struct.pb.h +++ b/src/google/protobuf/struct.pb.h @@ -8,12 +8,12 @@ #include <google/protobuf/stubs/common.h> -#if GOOGLE_PROTOBUF_VERSION < 3005000 +#if GOOGLE_PROTOBUF_VERSION < 3006000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3005001 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION +#if 3006000 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/stubs/bytestream.h b/src/google/protobuf/stubs/bytestream.h index b9b43db0..cfa19f29 100644 --- a/src/google/protobuf/stubs/bytestream.h +++ b/src/google/protobuf/stubs/bytestream.h @@ -57,7 +57,6 @@ #include <google/protobuf/stubs/stringpiece.h> class CordByteSink; -class MemBlock; namespace google { namespace protobuf { diff --git a/src/google/protobuf/stubs/common.cc b/src/google/protobuf/stubs/common.cc index 33d24c57..6544c6ed 100644 --- a/src/google/protobuf/stubs/common.cc +++ b/src/google/protobuf/stubs/common.cc @@ -339,66 +339,42 @@ namespace internal { typedef void OnShutdownFunc(); struct ShutdownData { ~ShutdownData() { - for (int i = 0; i < functions.size(); i++) { - functions[i](); - } - for (int i = 0; i < strings.size(); i++) { - strings[i]->~string(); - } - for (int i = 0; i < messages.size(); i++) { - messages[i]->~MessageLite(); - } + std::reverse(functions.begin(), functions.end()); + for (auto pair : functions) pair.first(pair.second); } - std::vector<void (*)()> functions; - std::vector<const std::string*> strings; - std::vector<const MessageLite*> messages; + static ShutdownData* get() { + static auto* data = new ShutdownData; + return data; + } + + std::vector<std::pair<void (*)(const void*), const void*>> functions; Mutex mutex; }; -ShutdownData* shutdown_data = NULL; -GOOGLE_PROTOBUF_DECLARE_ONCE(shutdown_functions_init); - -void InitShutdownFunctions() { - shutdown_data = new ShutdownData; -} - -inline void InitShutdownFunctionsOnce() { - GoogleOnceInit(&shutdown_functions_init, &InitShutdownFunctions); +static void RunZeroArgFunc(const void* arg) { + reinterpret_cast<void (*)()>(const_cast<void*>(arg))(); } void OnShutdown(void (*func)()) { - InitShutdownFunctionsOnce(); - MutexLock lock(&shutdown_data->mutex); - shutdown_data->functions.push_back(func); + OnShutdownRun(RunZeroArgFunc, reinterpret_cast<void*>(func)); } -void OnShutdownDestroyString(const std::string* ptr) { - InitShutdownFunctionsOnce(); +void OnShutdownRun(void (*f)(const void*), const void* arg) { + auto shutdown_data = ShutdownData::get(); MutexLock lock(&shutdown_data->mutex); - shutdown_data->strings.push_back(ptr); -} - -void OnShutdownDestroyMessage(const void* ptr) { - InitShutdownFunctionsOnce(); - MutexLock lock(&shutdown_data->mutex); - shutdown_data->messages.push_back(static_cast<const MessageLite*>(ptr)); + shutdown_data->functions.push_back(std::make_pair(f, arg)); } } // namespace internal void ShutdownProtobufLibrary() { - internal::InitShutdownFunctionsOnce(); - - // We don't need to lock shutdown_functions_mutex because it's up to the - // caller to make sure that no one is using the library before this is - // called. - - // Make it safe to call this multiple times. - if (internal::shutdown_data == NULL) return; - - delete internal::shutdown_data; - internal::shutdown_data = NULL; + // This function should be called only once, but accepts multiple calls. + static bool is_shutdown = false; + if (!is_shutdown) { + delete internal::ShutdownData::get(); + is_shutdown = true; + } } #if PROTOBUF_USE_EXCEPTIONS diff --git a/src/google/protobuf/stubs/common.h b/src/google/protobuf/stubs/common.h index c336383d..c5256d32 100644 --- a/src/google/protobuf/stubs/common.h +++ b/src/google/protobuf/stubs/common.h @@ -101,27 +101,27 @@ namespace internal { // The current version, represented as a single integer to make comparison // easier: major * 10^6 + minor * 10^3 + micro -#define GOOGLE_PROTOBUF_VERSION 3005001 +#define GOOGLE_PROTOBUF_VERSION 3006000 // A suffix string for alpha, beta or rc releases. Empty for stable releases. #define GOOGLE_PROTOBUF_VERSION_SUFFIX "" // The minimum library version which works with the current version of the // headers. -#define GOOGLE_PROTOBUF_MIN_LIBRARY_VERSION 3005000 +#define GOOGLE_PROTOBUF_MIN_LIBRARY_VERSION 3006000 // The minimum header version which works with the current version of // the library. This constant should only be used by protoc's C++ code // generator. -static const int kMinHeaderVersionForLibrary = 3005000; +static const int kMinHeaderVersionForLibrary = 3006000; // The minimum protoc version which works with the current version of the // headers. -#define GOOGLE_PROTOBUF_MIN_PROTOC_VERSION 3005000 +#define GOOGLE_PROTOBUF_MIN_PROTOC_VERSION 3006000 // The minimum header version which works with the current version of // protoc. This constant should only be used in VerifyVersion(). -static const int kMinHeaderVersionForProtoc = 3005000; +static const int kMinHeaderVersionForProtoc = 3006000; // Verifies that the headers and libraries are compatible. Use the macro // below to call this. @@ -193,17 +193,22 @@ LIBPROTOBUF_EXPORT char* UTF8CoerceToStructurallyValid( // // It is safe to call this multiple times. However, it is not safe to use // any other part of the protocol buffers library after -// ShutdownProtobufLibrary() has been called. +// ShutdownProtobufLibrary() has been called. Furthermore this call is not +// thread safe, user needs to synchronize multiple calls. LIBPROTOBUF_EXPORT void ShutdownProtobufLibrary(); namespace internal { // Register a function to be called when ShutdownProtocolBuffers() is called. LIBPROTOBUF_EXPORT void OnShutdown(void (*func)()); -// Destroy the string (call string destructor) -LIBPROTOBUF_EXPORT void OnShutdownDestroyString(const std::string* ptr); -// Destroy (not delete) the message -LIBPROTOBUF_EXPORT void OnShutdownDestroyMessage(const void* ptr); +// Run an arbitrary function on an arg +LIBPROTOBUF_EXPORT void OnShutdownRun(void (*f)(const void*), const void* arg); + +template <typename T> +T* OnShutdownDelete(T* p) { + OnShutdownRun([](const void* pp) { delete static_cast<const T*>(pp); }, p); + return p; +} } // namespace internal diff --git a/src/google/protobuf/stubs/platform_macros.h b/src/google/protobuf/stubs/platform_macros.h index ff4dea7b..ce1b1e36 100644 --- a/src/google/protobuf/stubs/platform_macros.h +++ b/src/google/protobuf/stubs/platform_macros.h @@ -56,7 +56,7 @@ #elif defined(__aarch64__) #define GOOGLE_PROTOBUF_ARCH_AARCH64 1 #define GOOGLE_PROTOBUF_ARCH_64_BIT 1 -#elif defined(__MIPSEL__) +#elif defined(__mips__) #if defined(__LP64__) #define GOOGLE_PROTOBUF_ARCH_MIPS64 1 #define GOOGLE_PROTOBUF_ARCH_64_BIT 1 diff --git a/src/google/protobuf/text_format_unittest.cc b/src/google/protobuf/text_format_unittest.cc index 544c37e2..53184130 100644 --- a/src/google/protobuf/text_format_unittest.cc +++ b/src/google/protobuf/text_format_unittest.cc @@ -574,6 +574,11 @@ class CompactRepeatedFieldPrinter : public TextFormat::FastFieldValuePrinter { generator->PrintString(field->name()); } } + // To prevent compiler complaining about Woverloaded-virtual + void PrintFieldName(const Message& message, + const Reflection* reflection, + const FieldDescriptor* field, + TextFormat::BaseTextGenerator* generator) const override {} void PrintMessageStart( const Message& message, int field_index, int field_count, bool single_line_mode, diff --git a/src/google/protobuf/timestamp.pb.cc b/src/google/protobuf/timestamp.pb.cc index 3184bd97..4269031f 100644 --- a/src/google/protobuf/timestamp.pb.cc +++ b/src/google/protobuf/timestamp.pb.cc @@ -167,7 +167,7 @@ void Timestamp::ArenaDtor(void* object) { Timestamp* _this = reinterpret_cast< Timestamp* >(object); (void)_this; } -void Timestamp::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void Timestamp::RegisterArenaDtor(::google::protobuf::Arena*) { } void Timestamp::SetCachedSize(int size) const { _cached_size_.Set(size); diff --git a/src/google/protobuf/timestamp.pb.h b/src/google/protobuf/timestamp.pb.h index f73e75b3..daea80f6 100644 --- a/src/google/protobuf/timestamp.pb.h +++ b/src/google/protobuf/timestamp.pb.h @@ -8,12 +8,12 @@ #include <google/protobuf/stubs/common.h> -#if GOOGLE_PROTOBUF_VERSION < 3005000 +#if GOOGLE_PROTOBUF_VERSION < 3006000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3005001 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION +#if 3006000 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/type.pb.cc b/src/google/protobuf/type.pb.cc index 17d5ce2a..72679df8 100644 --- a/src/google/protobuf/type.pb.cc +++ b/src/google/protobuf/type.pb.cc @@ -492,7 +492,7 @@ void Type::ArenaDtor(void* object) { Type* _this = reinterpret_cast< Type* >(object); (void)_this; } -void Type::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void Type::RegisterArenaDtor(::google::protobuf::Arena*) { } void Type::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -609,7 +609,7 @@ bool Type::MergePartialFromCodedStream( case 6: { if (static_cast< ::google::protobuf::uint8>(tag) == static_cast< ::google::protobuf::uint8>(48u /* 48 & 0xFF */)) { - int value; + int value = 0; DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( input, &value))); @@ -1014,7 +1014,7 @@ void Field::ArenaDtor(void* object) { Field* _this = reinterpret_cast< Field* >(object); (void)_this; } -void Field::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void Field::RegisterArenaDtor(::google::protobuf::Arena*) { } void Field::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -1061,7 +1061,7 @@ bool Field::MergePartialFromCodedStream( case 1: { if (static_cast< ::google::protobuf::uint8>(tag) == static_cast< ::google::protobuf::uint8>(8u /* 8 & 0xFF */)) { - int value; + int value = 0; DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( input, &value))); @@ -1076,7 +1076,7 @@ bool Field::MergePartialFromCodedStream( case 2: { if (static_cast< ::google::protobuf::uint8>(tag) == static_cast< ::google::protobuf::uint8>(16u /* 16 & 0xFF */)) { - int value; + int value = 0; DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( input, &value))); @@ -1698,7 +1698,7 @@ void Enum::ArenaDtor(void* object) { Enum* _this = reinterpret_cast< Enum* >(object); (void)_this; } -void Enum::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void Enum::RegisterArenaDtor(::google::protobuf::Arena*) { } void Enum::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -1797,7 +1797,7 @@ bool Enum::MergePartialFromCodedStream( case 5: { if (static_cast< ::google::protobuf::uint8>(tag) == static_cast< ::google::protobuf::uint8>(40u /* 40 & 0xFF */)) { - int value; + int value = 0; DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( input, &value))); @@ -2140,7 +2140,7 @@ void EnumValue::ArenaDtor(void* object) { EnumValue* _this = reinterpret_cast< EnumValue* >(object); (void)_this; } -void EnumValue::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void EnumValue::RegisterArenaDtor(::google::protobuf::Arena*) { } void EnumValue::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -2520,7 +2520,7 @@ void Option::ArenaDtor(void* object) { Option* _this = reinterpret_cast< Option* >(object); (void)_this; } -void Option::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void Option::RegisterArenaDtor(::google::protobuf::Arena*) { } void Option::SetCachedSize(int size) const { _cached_size_.Set(size); diff --git a/src/google/protobuf/type.pb.h b/src/google/protobuf/type.pb.h index 69af6c2e..cbd0cdcc 100644 --- a/src/google/protobuf/type.pb.h +++ b/src/google/protobuf/type.pb.h @@ -8,12 +8,12 @@ #include <google/protobuf/stubs/common.h> -#if GOOGLE_PROTOBUF_VERSION < 3005000 +#if GOOGLE_PROTOBUF_VERSION < 3006000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3005001 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION +#if 3006000 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/unknown_field_set.cc b/src/google/protobuf/unknown_field_set.cc index 0ada85e5..35f24e7a 100644 --- a/src/google/protobuf/unknown_field_set.cc +++ b/src/google/protobuf/unknown_field_set.cc @@ -46,28 +46,9 @@ namespace google { namespace protobuf { -namespace { -// This global instance is returned by unknown_fields() on any message class -// when the object has no unknown fields. This is necessary because we now -// instantiate the UnknownFieldSet dynamically only when required. -UnknownFieldSet* default_unknown_field_set_instance_ = NULL; - -void DeleteDefaultUnknownFieldSet() { - delete default_unknown_field_set_instance_; -} - -void InitDefaultUnknownFieldSet() { - default_unknown_field_set_instance_ = new UnknownFieldSet(); - internal::OnShutdown(&DeleteDefaultUnknownFieldSet); -} - -GOOGLE_PROTOBUF_DECLARE_ONCE(default_unknown_field_set_once_init_); -} - const UnknownFieldSet* UnknownFieldSet::default_instance() { - ::google::protobuf::GoogleOnceInit(&default_unknown_field_set_once_init_, - &InitDefaultUnknownFieldSet); - return default_unknown_field_set_instance_; + static auto instance = internal::OnShutdownDelete(new UnknownFieldSet()); + return instance; } void UnknownFieldSet::ClearFallback() { diff --git a/src/google/protobuf/wrappers.pb.cc b/src/google/protobuf/wrappers.pb.cc index 386be61d..8f254b82 100644 --- a/src/google/protobuf/wrappers.pb.cc +++ b/src/google/protobuf/wrappers.pb.cc @@ -391,7 +391,7 @@ void DoubleValue::ArenaDtor(void* object) { DoubleValue* _this = reinterpret_cast< DoubleValue* >(object); (void)_this; } -void DoubleValue::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void DoubleValue::RegisterArenaDtor(::google::protobuf::Arena*) { } void DoubleValue::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -642,7 +642,7 @@ void FloatValue::ArenaDtor(void* object) { FloatValue* _this = reinterpret_cast< FloatValue* >(object); (void)_this; } -void FloatValue::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void FloatValue::RegisterArenaDtor(::google::protobuf::Arena*) { } void FloatValue::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -893,7 +893,7 @@ void Int64Value::ArenaDtor(void* object) { Int64Value* _this = reinterpret_cast< Int64Value* >(object); (void)_this; } -void Int64Value::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void Int64Value::RegisterArenaDtor(::google::protobuf::Arena*) { } void Int64Value::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -1146,7 +1146,7 @@ void UInt64Value::ArenaDtor(void* object) { UInt64Value* _this = reinterpret_cast< UInt64Value* >(object); (void)_this; } -void UInt64Value::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void UInt64Value::RegisterArenaDtor(::google::protobuf::Arena*) { } void UInt64Value::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -1399,7 +1399,7 @@ void Int32Value::ArenaDtor(void* object) { Int32Value* _this = reinterpret_cast< Int32Value* >(object); (void)_this; } -void Int32Value::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void Int32Value::RegisterArenaDtor(::google::protobuf::Arena*) { } void Int32Value::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -1652,7 +1652,7 @@ void UInt32Value::ArenaDtor(void* object) { UInt32Value* _this = reinterpret_cast< UInt32Value* >(object); (void)_this; } -void UInt32Value::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void UInt32Value::RegisterArenaDtor(::google::protobuf::Arena*) { } void UInt32Value::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -1905,7 +1905,7 @@ void BoolValue::ArenaDtor(void* object) { BoolValue* _this = reinterpret_cast< BoolValue* >(object); (void)_this; } -void BoolValue::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void BoolValue::RegisterArenaDtor(::google::protobuf::Arena*) { } void BoolValue::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -2161,7 +2161,7 @@ void StringValue::ArenaDtor(void* object) { StringValue* _this = reinterpret_cast< StringValue* >(object); (void)_this; } -void StringValue::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void StringValue::RegisterArenaDtor(::google::protobuf::Arena*) { } void StringValue::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -2433,7 +2433,7 @@ void BytesValue::ArenaDtor(void* object) { BytesValue* _this = reinterpret_cast< BytesValue* >(object); (void)_this; } -void BytesValue::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void BytesValue::RegisterArenaDtor(::google::protobuf::Arena*) { } void BytesValue::SetCachedSize(int size) const { _cached_size_.Set(size); diff --git a/src/google/protobuf/wrappers.pb.h b/src/google/protobuf/wrappers.pb.h index 9f51a062..b00d4ea9 100644 --- a/src/google/protobuf/wrappers.pb.h +++ b/src/google/protobuf/wrappers.pb.h @@ -8,12 +8,12 @@ #include <google/protobuf/stubs/common.h> -#if GOOGLE_PROTOBUF_VERSION < 3005000 +#if GOOGLE_PROTOBUF_VERSION < 3006000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3005001 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION +#if 3006000 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. @@ -1,14 +1,8 @@ #!/bin/bash # -# Build and runs tests for the protobuf project. The tests as written here are -# used by both Jenkins and Travis, though some specialized logic is required to -# handle the differences between them. - -on_travis() { - if [ "$TRAVIS" == "true" ]; then - "$@" - fi -} +# Build and runs tests for the protobuf project. We use this script to run +# tests on kokoro (Ubuntu and MacOS). It can run locally as well but you +# will need to make sure the required compilers/tools are available. # For when some other test needs the C++ main build, including protoc and # libprotobuf. @@ -18,15 +12,6 @@ internal_build_cpp() { return fi - if [[ $(uname -s) == "Linux" && "$TRAVIS" == "true" ]]; then - # Install GCC 4.8 to replace the default GCC 4.6. We need 4.8 for more - # decent C++ 11 support in order to compile conformance tests. - sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y - sudo apt-get update -qq - sudo apt-get install -qq g++-4.8 - export CXX="g++-4.8" CC="gcc-4.8" - fi - # Initialize any submodules. git submodule update --init --recursive @@ -43,8 +28,6 @@ build_cpp() { # The benchmark code depends on cmake, so test if it is installed before # trying to do the build. - # NOTE: The travis macOS images say they have cmake, but the xcode8.1 image - # appears to be missing it: https://github.com/travis-ci/travis-ci/issues/6996 if [[ $(type cmake 2>/dev/null) ]]; then # Verify benchmarking code can build successfully. cd benchmarks && make cpp-benchmark && cd .. @@ -121,10 +104,6 @@ build_golang() { # Add protoc to the path so that the examples build finds it. export PATH="`pwd`/src:$PATH" - # Install Go and the Go protobuf compiler plugin. - on_travis sudo apt-get update -qq - on_travis sudo apt-get install -qq golang - export GOPATH="$HOME/gocode" mkdir -p "$GOPATH/src/github.com/google" rm -f "$GOPATH/src/github.com/google/protobuf" @@ -139,27 +118,17 @@ use_java() { version=$1 case "$version" in jdk7) - on_travis sudo apt-get install openjdk-7-jdk export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 ;; oracle7) - if [ "$TRAVIS" == "true" ]; then - sudo apt-get install python-software-properties # for apt-add-repository - echo "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" | \ - sudo debconf-set-selections - yes | sudo apt-add-repository ppa:webupd8team/java - yes | sudo apt-get install oracle-java7-installer - fi; export PATH=/usr/lib/jvm/java-7-oracle/bin:$PATH export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 ;; esac - if [ "$TRAVIS" != "true" ]; then - MAVEN_LOCAL_REPOSITORY=/var/maven_local_repository - MVN="$MVN -e -X --offline -Dmaven.repo.local=$MAVEN_LOCAL_REPOSITORY" - fi; + MAVEN_LOCAL_REPOSITORY=/var/maven_local_repository + MVN="$MVN -e -X --offline -Dmaven.repo.local=$MAVEN_LOCAL_REPOSITORY" which java java -version @@ -190,12 +159,6 @@ build_java_with_conformance_tests() { cd conformance && make test_java && cd .. } -build_javanano() { - # Java build needs `protoc`. - internal_build_cpp - cd javanano && $MVN test && cd .. -} - build_java_jdk7() { use_java jdk7 build_java_with_conformance_tests @@ -213,38 +176,6 @@ build_java_compatibility() { ./test.sh 3.0.0-beta-4 } -build_javanano_jdk7() { - use_java jdk7 - build_javanano -} -build_javanano_oracle7() { - use_java oracle7 - build_javanano -} - -internal_install_python_deps() { - if [ "$TRAVIS" != "true" ]; then - return; - fi - # Install tox (OS X doesn't have pip). - if [ $(uname -s) == "Darwin" ]; then - brew upgrade python - python3 -m pip install tox - else - sudo pip install tox - fi - # Only install Python2.6/3.x on Linux. - if [ $(uname -s) == "Linux" ]; then - sudo apt-get install -y python-software-properties # for apt-add-repository - sudo apt-add-repository -y ppa:fkrull/deadsnakes - sudo apt-get update -qq - sudo apt-get install -y python3.3 python3.3-dev - sudo apt-get install -y python3.4 python3.4-dev - sudo apt-get install -y python3.5 python3.5-dev - sudo apt-get install -y python3.6 python3.6-dev - fi -} - build_objectivec_ios() { # Reused the build script that takes care of configuring and ensuring things # are up to date. The OS X test runs the objc conformance test, so skip it @@ -279,9 +210,7 @@ build_objectivec_cocoapods_integration() { build_python() { internal_build_cpp - internal_install_python_deps cd python - # Only test Python 2.6/3.x on Linux if [ $(uname -s) == "Linux" ]; then envlist=py\{27,33,34,35,36\}-python else @@ -293,11 +222,9 @@ build_python() { build_python_cpp() { internal_build_cpp - internal_install_python_deps export LD_LIBRARY_PATH=../src/.libs # for Linux export DYLD_LIBRARY_PATH=../src/.libs # for OS X cd python - # Only test Python 3.x on Linux if [ $(uname -s) == "Linux" ]; then envlist=py\{27,33,34,35,36\}-cpp else @@ -325,18 +252,24 @@ build_ruby22() { internal_build_cpp # For conformance tests. cd ruby && bash travis-test.sh ruby-2.2 && cd .. } -build_jruby() { +build_ruby23() { + internal_build_cpp # For conformance tests. + cd ruby && bash travis-test.sh ruby-2.3 && cd .. +} +build_ruby24() { internal_build_cpp # For conformance tests. - # TODO(xiaofeng): Upgrade to jruby-9.x. There are some broken jests to be - # fixed. - cd ruby && bash travis-test.sh jruby-1.7 && cd .. + cd ruby && bash travis-test.sh ruby-2.4 && cd .. +} +build_ruby25() { + internal_build_cpp # For conformance tests. + cd ruby && bash travis-test.sh ruby-2.5.0 && cd .. } build_ruby_all() { build_ruby21 build_ruby22 - # TODO(teboring): Disable jruby test temperarily for it randomly fails. - # https://grpc-testing.appspot.com/job/protobuf_pull_request/735/consoleFull. - # build_jruby + build_ruby23 + build_ruby24 + build_ruby25 } build_javascript() { @@ -617,14 +550,6 @@ build_php_all() { build_php_compatibility } -# Note: travis currently does not support testing more than one language so the -# .travis.yml cheats and claims to only be cpp. If they add multiple language -# support, this should probably get updated to install steps and/or -# rvm/gemfile/jdk/etc. entries rather than manually doing the work. - -# .travis.yml uses matrix.exclude to block the cases where app-get can't be -# use to install things. - # -------- main -------- if [ "$#" -ne 1 ]; then @@ -635,8 +560,6 @@ Usage: $0 { cpp | java_jdk7 | java_oracle7 | java_compatibility | - javanano_jdk7 | - javanano_oracle7 | objectivec_ios | objectivec_ios_debug | objectivec_ios_release | @@ -665,4 +588,5 @@ fi set -e # exit immediately on error set -x # display all commands +cd $(dirname $0) eval "build_$1" |