diff options
28 files changed, 723 insertions, 284 deletions
diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 67d433bc..7e1b385e 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -1,15 +1,19 @@ # Minimum CMake required -cmake_minimum_required(VERSION 2.8) +cmake_minimum_required(VERSION 2.8.12) -# Project -project(protobuf C CXX) +if(protobuf_VERBOSE) + message(STATUS "Protocol Buffers Configuring...") +endif() # CMake policies cmake_policy(SET CMP0022 NEW) +# Project +project(protobuf C CXX) + # Options -option(protobuf_VERBOSE "Enable for verbose output" OFF) option(protobuf_BUILD_TESTS "Build tests" ON) +option(protobuf_BUILD_EXAMPLES "Build examples" OFF) if (BUILD_SHARED_LIBS) set(protobuf_BUILD_SHARED_LIBS_DEFAULT ON) else (BUILD_SHARED_LIBS) @@ -25,6 +29,9 @@ endif (MSVC) option(protobuf_WITH_ZLIB "Build with zlib support" ${protobuf_WITH_ZLIB_DEFAULT}) set(protobuf_DEBUG_POSTFIX "d" CACHE STRING "Default debug postfix") +mark_as_advanced(protobuf_DEBUG_POSTFIX) +# User options +include(protobuf-options.cmake) # Path to main configure script set(protobuf_CONFIGURE_SCRIPT "../configure.ac") @@ -158,3 +165,11 @@ if (protobuf_BUILD_TESTS) endif (protobuf_BUILD_TESTS) include(install.cmake) + +if (protobuf_BUILD_EXAMPLES) + include(examples.cmake) +endif (protobuf_BUILD_EXAMPLES) + +if(protobuf_VERBOSE) + message(STATUS "Protocol Buffers Configuring done") +endif() diff --git a/cmake/examples.cmake b/cmake/examples.cmake new file mode 100644 index 00000000..5a2538e4 --- /dev/null +++ b/cmake/examples.cmake @@ -0,0 +1,57 @@ +if(protobuf_VERBOSE)
+ message(STATUS "Protocol Buffers Examples Configuring...")
+endif()
+
+get_filename_component(examples_dir "../examples" ABSOLUTE)
+
+if(protobuf_VERBOSE)
+ message(STATUS "Protocol Buffers Examples Configuring done")
+endif()
+include(ExternalProject)
+
+# Internal utility function: Create a custom target representing a build of examples with custom options.
+function(add_examples_build NAME)
+ ExternalProject_Add(${NAME}
+ PREFIX ${NAME}
+ SOURCE_DIR "${examples_dir}"
+ BINARY_DIR ${NAME}
+ STAMP_DIR ${NAME}/logs
+ INSTALL_COMMAND "" #Skip
+ LOG_CONFIGURE 1
+ CMAKE_CACHE_ARGS "-Dprotobuf_MSVC_STATIC_RUNTIME:BOOL=${protobuf_MSVC_STATIC_RUNTIME}"
+ "-DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}"
+ "-Dprotobuf_VERBOSE:BOOL=${protobuf_VERBOSE}"
+ ${ARGN}
+ )
+ set_property(TARGET ${NAME} PROPERTY FOLDER "Examples")
+ set_property(TARGET ${NAME} PROPERTY EXCLUDE_FROM_ALL TRUE)
+endfunction()
+
+# Add examples as an external project.
+# sub_directory cannot be used because the find_package(protobuf) call would cause failures with redefined targets.
+add_examples_build(examples "-Dprotobuf_DIR:PATH=${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_CMAKEDIR}")
+add_dependencies(examples libprotobuf protoc)
+
+option(protobuf_BUILD_EXAMPLES_MULTITEST "Build Examples in multiple configurations. Useful for testing." OFF)
+mark_as_advanced(protobuf_BUILD_EXAMPLES_MULTITEST)
+if(protobuf_BUILD_EXAMPLES_MULTITEST)
+ set_property(GLOBAL PROPERTY USE_FOLDERS ON)
+
+ #Build using the legacy compatiblity module.
+ add_examples_build(examples-legacy
+ "-Dprotobuf_DIR:PATH=${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_CMAKEDIR}"
+ "-Dprotobuf_MODULE_COMPATIBLE:BOOL=TRUE"
+ )
+ add_dependencies(examples-legacy libprotobuf protoc)
+
+ #Build using the installed library.
+ add_examples_build(examples-installed
+ "-Dprotobuf_DIR:PATH=${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_CMAKEDIR}"
+ )
+
+ #Build using the installed library in legacy compatiblity mode.
+ add_examples_build(examples-installed-legacy
+ "-Dprotobuf_DIR:PATH=${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_CMAKEDIR}"
+ "-Dprotobuf_MODULE_COMPATIBLE:BOOL=TRUE"
+ )
+endif()
diff --git a/cmake/extract_includes.bat.in b/cmake/extract_includes.bat.in index b593e0c9..c76973c9 100644 --- a/cmake/extract_includes.bat.in +++ b/cmake/extract_includes.bat.in @@ -23,6 +23,7 @@ copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\command_line_ copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\cpp\cpp_generator.h include\google\protobuf\compiler\cpp\cpp_generator.h copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\csharp\csharp_generator.h include\google\protobuf\compiler\csharp\csharp_generator.h copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\csharp\csharp_names.h include\google\protobuf\compiler\csharp\csharp_names.h +copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\csharp\csharp_options.h include\google\protobuf\compiler\csharp\csharp_options.h copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\importer.h include\google\protobuf\compiler\importer.h copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\java\java_generator.h include\google\protobuf\compiler\java\java_generator.h copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\java\java_names.h include\google\protobuf\compiler\java\java_names.h diff --git a/cmake/install.cmake b/cmake/install.cmake index a5040b22..94ef2198 100644 --- a/cmake/install.cmake +++ b/cmake/install.cmake @@ -6,6 +6,7 @@ foreach(_library libprotoc) set_property(TARGET ${_library} PROPERTY INTERFACE_INCLUDE_DIRECTORIES + $<BUILD_INTERFACE:${protobuf_source_dir}/src> $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>) install(TARGETS ${_library} EXPORT protobuf-targets RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT ${_library} @@ -80,29 +81,43 @@ foreach(_file ${nobase_dist_proto_DATA}) endif() endforeach() -# Export configuration +# Install configuration set(_cmakedir_desc "Directory relative to CMAKE_INSTALL to install the cmake configuration files") if(NOT MSVC) set(CMAKE_INSTALL_CMAKEDIR "${CMAKE_INSTALL_LIBDIR}/cmake/protobuf" CACHE STRING "${_cmakedir_desc}") else() set(CMAKE_INSTALL_CMAKEDIR "cmake" CACHE STRING "${_cmakedir_desc}") endif() - -install(EXPORT protobuf-targets - DESTINATION "${CMAKE_INSTALL_CMAKEDIR}" - NAMESPACE protobuf:: - COMPONENT protobuf-export) +mark_as_advanced(CMAKE_INSTALL_CMAKEDIR) configure_file(protobuf-config.cmake.in - protobuf-config.cmake @ONLY) + ${CMAKE_INSTALL_CMAKEDIR}/protobuf-config.cmake @ONLY) configure_file(protobuf-config-version.cmake.in - protobuf-config-version.cmake @ONLY) + ${CMAKE_INSTALL_CMAKEDIR}/protobuf-config-version.cmake @ONLY) configure_file(protobuf-module.cmake.in - protobuf-module.cmake @ONLY) + ${CMAKE_INSTALL_CMAKEDIR}/protobuf-module.cmake @ONLY) +configure_file(protobuf-options.cmake + ${CMAKE_INSTALL_CMAKEDIR}/protobuf-options.cmake @ONLY) + +# Allows the build directory to be used as a find directory. +export(TARGETS libprotobuf-lite libprotobuf libprotoc protoc + NAMESPACE protobuf:: + FILE ${CMAKE_INSTALL_CMAKEDIR}/protobuf-targets.cmake +) -install(FILES - "${protobuf_BINARY_DIR}/protobuf-config.cmake" - "${protobuf_BINARY_DIR}/protobuf-config-version.cmake" - "${protobuf_BINARY_DIR}/protobuf-module.cmake" +install(EXPORT protobuf-targets DESTINATION "${CMAKE_INSTALL_CMAKEDIR}" + NAMESPACE protobuf:: COMPONENT protobuf-export) + +install(DIRECTORY ${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_CMAKEDIR}/ + DESTINATION "${CMAKE_INSTALL_CMAKEDIR}" + COMPONENT protobuf-export + PATTERN protobuf-targets.cmake EXCLUDE +) + +option(protobuf_INSTALL_EXAMPLES "Install the examples folder" OFF) +if(protobuf_INSTALL_EXAMPLES) + install(DIRECTORY ../examples/ DESTINATION examples + COMPONENT protobuf-examples) +endif() diff --git a/cmake/protobuf-config.cmake.in b/cmake/protobuf-config.cmake.in index f11796a4..37315510 100644 --- a/cmake/protobuf-config.cmake.in +++ b/cmake/protobuf-config.cmake.in @@ -1,19 +1,10 @@ -# Version info variables -set(PROTOBUF_VERSION "@protobuf_VERSION@") -set(PROTOBUF_VERSION_STRING "@protobuf_VERSION_STRING@") +# User options +include("${CMAKE_CURRENT_LIST_DIR}/protobuf-options.cmake") # Imported targets include("${CMAKE_CURRENT_LIST_DIR}/protobuf-targets.cmake") -# Compute the installation prefix relative to this file. -get_filename_component(_PROTOBUF_IMPORT_PREFIX - "${_PROTOBUF_PACKAGE_PREFIX}" PATH) -get_filename_component(_PROTOBUF_IMPORT_PREFIX - "${_PROTOBUF_IMPORT_PREFIX}" PATH) -get_filename_component(_PROTOBUF_IMPORT_PREFIX - "${_PROTOBUF_IMPORT_PREFIX}" PATH) - # CMake FindProtobuf module compatible file -if(NOT DEFINED PROTOBUF_MODULE_COMPATIBLE OR "${PROTOBUF_MODULE_COMPATIBLE}") - include("${_PROTOBUF_PACKAGE_PREFIX}/protobuf-module.cmake") +if(protobuf_MODULE_COMPATIBLE) + include("${CMAKE_CURRENT_LIST_DIR}/protobuf-module.cmake") endif() diff --git a/cmake/protobuf-module.cmake.in b/cmake/protobuf-module.cmake.in index d81dc459..7ced934a 100644 --- a/cmake/protobuf-module.cmake.in +++ b/cmake/protobuf-module.cmake.in @@ -1,7 +1,4 @@ -if(PROTOBUF_SRC_ROOT_FOLDER) - message(AUTHOR_WARNING "Variable PROTOBUF_SRC_ROOT_FOLDER defined, but not" - " used in CONFIG mode") -endif() +# Functions function(PROTOBUF_GENERATE_CPP SRCS HDRS) if(NOT ARGN) @@ -23,12 +20,8 @@ function(PROTOBUF_GENERATE_CPP SRCS HDRS) set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR}) endif() - # Add well-known type protos include path - list(APPEND _protobuf_include_path - -I "${_PROTOBUF_IMPORT_PREFIX}/@CMAKE_INSTALL_INCLUDEDIR@") - - if(DEFINED PROTOBUF_IMPORT_DIRS) - foreach(DIR ${PROTOBUF_IMPORT_DIRS}) + if(DEFINED Protobuf_IMPORT_DIRS) + foreach(DIR ${Protobuf_IMPORT_DIRS}) get_filename_component(ABS_PATH ${DIR} ABSOLUTE) list(FIND _protobuf_include_path ${ABS_PATH} _contains_already) if(${_contains_already} EQUAL -1) @@ -49,11 +42,11 @@ function(PROTOBUF_GENERATE_CPP SRCS HDRS) add_custom_command( OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc" "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h" - COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} + COMMAND ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out ${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${ABS_FIL} - DEPENDS ${ABS_FIL} ${PROTOBUF_PROTOC_EXECUTABLE} + DEPENDS ${ABS_FIL} ${Protobuf_PROTOC_EXECUTABLE} COMMENT "Running C++ protocol buffer compiler on ${FIL}" - VERBATIM) + VERBATIM ) endforeach() set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE) @@ -61,29 +54,110 @@ function(PROTOBUF_GENERATE_CPP SRCS HDRS) set(${HDRS} ${${HDRS}} PARENT_SCOPE) endfunction() +function(PROTOBUF_GENERATE_PYTHON SRCS) + if(NOT ARGN) + message(SEND_ERROR "Error: PROTOBUF_GENERATE_PYTHON() called without any proto files") + return() + endif() + + if(PROTOBUF_GENERATE_CPP_APPEND_PATH) + # Create an include path for each file specified + foreach(FIL ${ARGN}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(ABS_PATH ${ABS_FIL} PATH) + list(FIND _protobuf_include_path ${ABS_PATH} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND _protobuf_include_path -I ${ABS_PATH}) + endif() + endforeach() + else() + set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR}) + endif() + + if(DEFINED Protobuf_IMPORT_DIRS) + foreach(DIR ${Protobuf_IMPORT_DIRS}) + get_filename_component(ABS_PATH ${DIR} ABSOLUTE) + list(FIND _protobuf_include_path ${ABS_PATH} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND _protobuf_include_path -I ${ABS_PATH}) + endif() + endforeach() + endif() + + set(${SRCS}) + foreach(FIL ${ARGN}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(FIL_WE ${FIL} NAME_WE) + + list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}_pb2.py") + add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}_pb2.py" + COMMAND ${Protobuf_PROTOC_EXECUTABLE} --python_out ${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${ABS_FIL} + DEPENDS ${ABS_FIL} ${Protobuf_PROTOC_EXECUTABLE} + COMMENT "Running Python protocol buffer compiler on ${FIL}" + VERBATIM ) + endforeach() + + set(${SRCS} ${${SRCS}} PARENT_SCOPE) +endfunction() + +# Environment + +# Backwards compatibility +# Define camel case versions of input variables +foreach(UPPER + PROTOBUF_SRC_ROOT_FOLDER + PROTOBUF_IMPORT_DIRS + PROTOBUF_DEBUG + PROTOBUF_LIBRARY + PROTOBUF_PROTOC_LIBRARY + PROTOBUF_INCLUDE_DIR + PROTOBUF_PROTOC_EXECUTABLE + PROTOBUF_LIBRARY_DEBUG + PROTOBUF_PROTOC_LIBRARY_DEBUG + PROTOBUF_LITE_LIBRARY + PROTOBUF_LITE_LIBRARY_DEBUG + ) + if (DEFINED ${UPPER}) + string(REPLACE "PROTOBUF_" "Protobuf_" Camel ${UPPER}) + if (NOT DEFINED ${Camel}) + set(${Camel} ${${UPPER}}) + endif() + endif() +endforeach() + +if(DEFINED Protobuf_SRC_ROOT_FOLDER) + message(AUTHOR_WARNING "Variable Protobuf_SRC_ROOT_FOLDER defined, but not" + " used in CONFIG mode") +endif() + +include(SelectLibraryConfigurations) + # Internal function: search for normal library as well as a debug one # if the debug one is specified also include debug/optimized keywords # in *_LIBRARIES variable function(_protobuf_find_libraries name filename) - get_target_property(${name}_LIBRARY lib${filename} - IMPORTED_LOCATION_RELEASE) - set(${name}_LIBRARY "${${name}_LIBRARY}" PARENT_SCOPE) - get_target_property(${name}_LIBRARY_DEBUG lib${filename} - IMPORTED_LOCATION_DEBUG) - set(${name}_LIBRARY_DEBUG "${${name}_LIBRARY_DEBUG}" PARENT_SCOPE) - - if(NOT ${name}_LIBRARY_DEBUG) - # There is no debug library - set(${name}_LIBRARY_DEBUG ${${name}_LIBRARY} PARENT_SCOPE) - set(${name}_LIBRARIES ${${name}_LIBRARY} PARENT_SCOPE) - else() - # There IS a debug library - set(${name}_LIBRARIES - optimized ${${name}_LIBRARY} - debug ${${name}_LIBRARY_DEBUG} - PARENT_SCOPE - ) - endif() + if(${name}_LIBRARIES) + # Use result recorded by a previous call. + elseif(${name}_LIBRARY) + # Honor cache entry used by CMake 3.5 and lower. + set(${name}_LIBRARIES "${${name}_LIBRARY}" PARENT_SCOPE) + else() + get_target_property(_aliased protobuf::lib${filename} ALIASED_TARGET) + if(_aliased) + set(${name}_LIBRARY_RELEASE $<TARGET_FILE:protobuf::lib${filename}>) + set(${name}_LIBRARY_DEBUG $<TARGET_FILE:protobuf::lib${filename}>) + else() + get_target_property(${name}_LIBRARY_RELEASE protobuf::lib${filename} + LOCATION_RELEASE) + get_target_property(${name}_LIBRARY_DEBUG protobuf::lib${filename} + LOCATION_DEBUG) + endif() + + select_library_configurations(${name}) + set(${name}_LIBRARY ${${name}_LIBRARY} PARENT_SCOPE) + set(${name}_LIBRARIES ${${name}_LIBRARIES} PARENT_SCOPE) + endif() endfunction() # Internal function: find threads library @@ -107,33 +181,69 @@ if(NOT DEFINED PROTOBUF_GENERATE_CPP_APPEND_PATH) endif() # The Protobuf library -_protobuf_find_libraries(PROTOBUF protobuf) +_protobuf_find_libraries(Protobuf protobuf) # The Protobuf Lite library -_protobuf_find_libraries(PROTOBUF_LITE protobuf-lite) +_protobuf_find_libraries(Protobuf_LITE protobuf-lite) # The Protobuf Protoc Library -_protobuf_find_libraries(PROTOBUF_PROTOC protoc) +_protobuf_find_libraries(Protobuf_PROTOC protoc) if(UNIX) _protobuf_find_threads() endif() # Set the include directory -set(PROTOBUF_INCLUDE_DIR "${_PROTOBUF_IMPORT_PREFIX}/@CMAKE_INSTALL_INCLUDEDIR@") +get_target_property(Protobuf_INCLUDE_DIRS protobuf::libprotobuf + INTERFACE_INCLUDE_DIRECTORIES) # Set the protoc Executable -get_target_property(PROTOBUF_PROTOC_EXECUTABLE protoc - IMPORTED_LOCATION_RELEASE) -if(NOT PROTOBUF_PROTOC_EXECUTABLE) - get_target_property(PROTOBUF_PROTOC_EXECUTABLE protoc - IMPORTED_LOCATION_DEBUG) +get_target_property(_aliased protobuf::protoc ALIASED_TARGET) +if(_aliased) + if(POLICY CMP0026) + set(Protobuf_PROTOC_EXECUTABLE $<TARGET_FILE:protobuf::protoc>) + else() + get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc + LOCATION) + endif() +else() + get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc + IMPORTED_LOCATION_RELEASE) + if(NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}") + get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc + IMPORTED_LOCATION_DEBUG) + endif() endif() -include(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(PROTOBUF DEFAULT_MSG - PROTOBUF_LIBRARY PROTOBUF_INCLUDE_DIR) +# Version info variable +set(Protobuf_VERSION "@protobuf_VERSION@") -if(PROTOBUF_FOUND) - set(PROTOBUF_INCLUDE_DIRS ${PROTOBUF_INCLUDE_DIR}) -endif() +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(Protobuf + REQUIRED_VARS Protobuf_LIBRARIES Protobuf_INCLUDE_DIRS + VERSION_VAR Protobuf_VERSION +) + +# Backwards compatibility +# Define upper case versions of output variables +foreach(Camel + Protobuf_VERSION + Protobuf_SRC_ROOT_FOLDER + Protobuf_IMPORT_DIRS + Protobuf_DEBUG + Protobuf_INCLUDE_DIRS + Protobuf_LIBRARIES + Protobuf_PROTOC_LIBRARIES + Protobuf_LITE_LIBRARIES + Protobuf_LIBRARY + Protobuf_PROTOC_LIBRARY + Protobuf_INCLUDE_DIR + Protobuf_PROTOC_EXECUTABLE + Protobuf_LIBRARY_DEBUG + Protobuf_PROTOC_LIBRARY_DEBUG + Protobuf_LITE_LIBRARY + Protobuf_LITE_LIBRARY_DEBUG + ) + string(TOUPPER ${Camel} UPPER) + set(${UPPER} ${${Camel}}) +endforeach() diff --git a/cmake/protobuf-options.cmake b/cmake/protobuf-options.cmake new file mode 100644 index 00000000..99c85ebe --- /dev/null +++ b/cmake/protobuf-options.cmake @@ -0,0 +1,7 @@ +# Verbose output
+option(protobuf_VERBOSE "Enable for verbose output" OFF)
+mark_as_advanced(protobuf_VERBOSE)
+
+# FindProtobuf module compatibel
+option(protobuf_MODULE_COMPATIBLE "CMake build-in FindProtobuf.cmake module compatible" OFF)
+mark_as_advanced(protobuf_MODULE_COMPATIBLE)
diff --git a/cmake/tests.cmake b/cmake/tests.cmake index 76fdf8ef..ec0d6030 100644 --- a/cmake/tests.cmake +++ b/cmake/tests.cmake @@ -4,6 +4,7 @@ endif() option(protobuf_ABSOLUTE_TEST_PLUGIN_PATH "Using absolute test_plugin path in tests" ON) +mark_as_advanced(protobuf_ABSOLUTE_TEST_PLUGIN_PATH) include_directories( ${protobuf_source_dir}/gmock diff --git a/configure.ac b/configure.ac index bdc72ee6..d858e177 100644 --- a/configure.ac +++ b/configure.ac @@ -4,8 +4,13 @@ AC_PREREQ(2.59) # Note: If you change the version, you must also update it in: -# * java/pom.xml -# * python/setup.py +# * Protobuf.podspec +# * csharp/Google.Protobuf.Tools.nuspec +# * csharp/src/*/AssemblyInfo.cs +# * csharp/src/Google.Protobuf/Google.Protobuf.nuspec +# * java/*/pom.xml +# * python/google/protobuf/__init__.py +# * protoc-artifacts/pom.xml # * src/google/protobuf/stubs/common.h # * src/Makefile.am (Update -version-info for LDFLAGS if needed) # diff --git a/conformance/Makefile.am b/conformance/Makefile.am index 31a9e408..5538cc75 100644 --- a/conformance/Makefile.am +++ b/conformance/Makefile.am @@ -268,10 +268,10 @@ test_ruby: protoc_middleman conformance-test-runner $(other_language_protoc_outp # These depend on library paths being properly set up. The easiest way to # run them is to just use "tox" from the python dir. test_python: protoc_middleman conformance-test-runner - ./conformance-test-runner --failure_list failure_list_python.txt $(CONFORMANCE_PYTHON_EXTRA_FAILURES) ./conformance_python.py + ./conformance-test-runner --failure_list failure_list_python.txt ./conformance_python.py test_python_cpp: protoc_middleman conformance-test-runner - ./conformance-test-runner --failure_list failure_list_python_cpp.txt $(CONFORMANCE_PYTHON_EXTRA_FAILURES) ./conformance_python.py + ./conformance-test-runner --failure_list failure_list_python_cpp.txt ./conformance_python.py if OBJC_CONFORMANCE_TEST diff --git a/conformance/conformance_test.cc b/conformance/conformance_test.cc index fc0605bf..59a61e51 100644 --- a/conformance/conformance_test.cc +++ b/conformance/conformance_test.cc @@ -30,6 +30,7 @@ #include <stdarg.h> #include <string> +#include <fstream> #include "conformance.pb.h" #include "conformance_test.h" @@ -575,24 +576,41 @@ void ConformanceTestSuite::TestPrematureEOFForType(FieldDescriptor::Type type) { } } -void ConformanceTestSuite::SetFailureList(const vector<string>& failure_list) { +void ConformanceTestSuite::SetFailureList(const string& filename, + const vector<string>& failure_list) { + failure_list_filename_ = filename; expected_to_fail_.clear(); std::copy(failure_list.begin(), failure_list.end(), std::inserter(expected_to_fail_, expected_to_fail_.end())); } bool ConformanceTestSuite::CheckSetEmpty(const set<string>& set_to_check, - const char* msg) { + const std::string& write_to_file, + const std::string& msg) { if (set_to_check.empty()) { return true; } else { StringAppendF(&output_, "\n"); - StringAppendF(&output_, "%s:\n", msg); + StringAppendF(&output_, "%s\n\n", msg.c_str()); for (set<string>::const_iterator iter = set_to_check.begin(); iter != set_to_check.end(); ++iter) { StringAppendF(&output_, " %s\n", iter->c_str()); } StringAppendF(&output_, "\n"); + + if (!write_to_file.empty()) { + std::ofstream os(write_to_file); + if (os) { + for (set<string>::const_iterator iter = set_to_check.begin(); + iter != set_to_check.end(); ++iter) { + os << *iter << "\n"; + } + } else { + StringAppendF(&output_, "Failed to open file: %s\n", + write_to_file.c_str()); + } + } + return false; } } @@ -1965,27 +1983,34 @@ bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner, )"); bool ok = true; - if (!CheckSetEmpty(expected_to_fail_, + if (!CheckSetEmpty(expected_to_fail_, "nonexistent_tests.txt", "These tests were listed in the failure list, but they " - "don't exist. Remove them from the failure list")) { + "don't exist. Remove them from the failure list by " + "running:\n" + " ./update_failure_list.py " + failure_list_filename_ + + " --remove nonexistent_tests.txt")) { ok = false; } - if (!CheckSetEmpty(unexpected_failing_tests_, + if (!CheckSetEmpty(unexpected_failing_tests_, "failing_tests.txt", "These tests failed. If they can't be fixed right now, " "you can add them to the failure list so the overall " - "suite can succeed")) { + "suite can succeed. Add them to the failure list by " + "running:\n" + " ./update_failure_list.py " + failure_list_filename_ + + " --add failing_tests.txt")) { + ok = false; + } + if (!CheckSetEmpty(unexpected_succeeding_tests_, "succeeding_tests.txt", + "These tests succeeded, even though they were listed in " + "the failure list. Remove them from the failure list " + "by running:\n" + " ./update_failure_list.py " + failure_list_filename_ + + " --remove succeeding_tests.txt")) { ok = false; } - - // Sometimes the testee may be fixed before we update the failure list (e.g., - // the testee is from a different component). We warn about this case but - // don't consider it an overall test failure. - CheckSetEmpty(unexpected_succeeding_tests_, - "These tests succeeded, even though they were listed in " - "the failure list. Remove them from the failure list"); if (verbose_) { - CheckSetEmpty(skipped_, + CheckSetEmpty(skipped_, "", "These tests were skipped (probably because support for some " "features is not implemented)"); } diff --git a/conformance/conformance_test.h b/conformance/conformance_test.h index 75fc97bc..c9c5213c 100644 --- a/conformance/conformance_test.h +++ b/conformance/conformance_test.h @@ -98,7 +98,11 @@ class ConformanceTestSuite { // Sets the list of tests that are expected to fail when RunSuite() is called. // RunSuite() will fail unless the set of failing tests is exactly the same // as this list. - void SetFailureList(const std::vector<std::string>& failure_list); + // + // The filename here is *only* used to create/format useful error messages for + // how to update the failure list. We do NOT read this file at all. + void SetFailureList(const std::string& filename, + const std::vector<std::string>& failure_list); // Run all the conformance tests against the given test runner. // Test output will be stored in "output". @@ -143,12 +147,14 @@ class ConformanceTestSuite { void ExpectHardParseFailureForProto(const std::string& proto, const std::string& test_name); void TestPrematureEOFForType(google::protobuf::FieldDescriptor::Type type); - bool CheckSetEmpty(const set<string>& set_to_check, const char* msg); + bool CheckSetEmpty(const set<string>& set_to_check, + const std::string& write_to_file, const std::string& msg); ConformanceTestRunner* runner_; int successes_; int expected_failures_; bool verbose_; std::string output_; + std::string failure_list_filename_; // The set of test names that are expected to fail in this run, but haven't // failed yet. diff --git a/conformance/conformance_test_runner.cc b/conformance/conformance_test_runner.cc index 376a60b9..d6b1175c 100644 --- a/conformance/conformance_test_runner.cc +++ b/conformance/conformance_test_runner.cc @@ -280,11 +280,13 @@ int main(int argc, char *argv[]) { char *program; google::protobuf::ConformanceTestSuite suite; + string failure_list_filename; vector<string> failure_list; for (int arg = 1; arg < argc; ++arg) { if (strcmp(argv[arg], "--failure_list") == 0) { if (++arg == argc) UsageError(); + failure_list_filename = argv[arg]; ParseFailureList(argv[arg], &failure_list); } else if (strcmp(argv[arg], "--verbose") == 0) { suite.SetVerbose(true); @@ -300,7 +302,7 @@ int main(int argc, char *argv[]) { } } - suite.SetFailureList(failure_list); + suite.SetFailureList(failure_list_filename, failure_list); ForkPipeRunner runner(program); std::string output; diff --git a/conformance/failure_list_cpp.txt b/conformance/failure_list_cpp.txt index 2ddf831c..839e5210 100644 --- a/conformance/failure_list_cpp.txt +++ b/conformance/failure_list_cpp.txt @@ -12,21 +12,11 @@ FieldMaskPathsDontRoundTrip.JsonOutput FieldMaskTooManyUnderscore.JsonOutput JsonInput.AnyUnorderedTypeTag.JsonOutput JsonInput.AnyUnorderedTypeTag.ProtobufOutput -JsonInput.AnyWithValueForInteger.JsonOutput -JsonInput.AnyWithValueForInteger.ProtobufOutput -JsonInput.AnyWithValueForJsonObject.JsonOutput -JsonInput.AnyWithValueForJsonObject.ProtobufOutput JsonInput.BoolFieldDoubleQuotedFalse JsonInput.BoolFieldDoubleQuotedTrue -JsonInput.BoolFieldIntegerOne -JsonInput.BoolFieldIntegerZero -JsonInput.BytesFieldInvalidBase64Characters JsonInput.BytesFieldNoPadding JsonInput.DoubleFieldTooSmall JsonInput.DurationHasZeroFractionalDigit.Validator -JsonInput.DurationJsonInputTooLarge -JsonInput.DurationJsonInputTooSmall -JsonInput.DurationMissingS JsonInput.EnumFieldUnknownValue.Validator JsonInput.FieldMaskInvalidCharacter JsonInput.FieldNameDuplicate @@ -36,58 +26,12 @@ JsonInput.FieldNameInLowerCamelCase.Validator JsonInput.FieldNameInSnakeCase.JsonOutput JsonInput.FieldNameInSnakeCase.ProtobufOutput JsonInput.FieldNameNotQuoted -JsonInput.FloatFieldTooLarge -JsonInput.FloatFieldTooSmall -JsonInput.Int32FieldLeadingSpace -JsonInput.Int32FieldLeadingZero -JsonInput.Int32FieldMinFloatValue.JsonOutput -JsonInput.Int32FieldMinFloatValue.ProtobufOutput -JsonInput.Int32FieldMinValue.JsonOutput -JsonInput.Int32FieldMinValue.ProtobufOutput -JsonInput.Int32FieldNegativeWithLeadingZero -JsonInput.Int32FieldNotInteger -JsonInput.Int32FieldNotNumber -JsonInput.Int32FieldTooLarge -JsonInput.Int32FieldTooSmall -JsonInput.Int32FieldTrailingSpace -JsonInput.Int64FieldNotInteger -JsonInput.Int64FieldNotNumber -JsonInput.Int64FieldTooLarge -JsonInput.Int64FieldTooSmall JsonInput.MapFieldValueIsNull -JsonInput.OneofFieldDuplicate JsonInput.RepeatedFieldMessageElementIsNull JsonInput.RepeatedFieldPrimitiveElementIsNull JsonInput.RepeatedFieldTrailingComma -JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool -JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotMessage -JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotString -JsonInput.RepeatedFieldWrongElementTypeExpectingMessagesGotBool -JsonInput.RepeatedFieldWrongElementTypeExpectingMessagesGotInt -JsonInput.RepeatedFieldWrongElementTypeExpectingMessagesGotString -JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool -JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt -JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotMessage -JsonInput.StringFieldNotAString -JsonInput.StringFieldSurrogateInWrongOrder -JsonInput.StringFieldSurrogatePair.JsonOutput -JsonInput.StringFieldSurrogatePair.ProtobufOutput -JsonInput.StringFieldUnpairedHighSurrogate -JsonInput.StringFieldUnpairedLowSurrogate JsonInput.StringFieldUppercaseEscapeLetter -JsonInput.TimestampJsonInputLowercaseT -JsonInput.TimestampJsonInputLowercaseZ -JsonInput.TimestampJsonInputMissingT -JsonInput.TimestampJsonInputMissingZ -JsonInput.TimestampJsonInputTooLarge -JsonInput.TimestampJsonInputTooSmall JsonInput.TrailingCommaInAnObject -JsonInput.Uint32FieldNotInteger -JsonInput.Uint32FieldNotNumber -JsonInput.Uint32FieldTooLarge -JsonInput.Uint64FieldNotInteger -JsonInput.Uint64FieldNotNumber -JsonInput.Uint64FieldTooLarge JsonInput.WrapperTypesWithNullValue.JsonOutput JsonInput.WrapperTypesWithNullValue.ProtobufOutput ProtobufInput.PrematureEofBeforeKnownRepeatedValue.MESSAGE @@ -102,5 +46,3 @@ ProtobufInput.PrematureEofInPackedField.SINT64 ProtobufInput.PrematureEofInPackedField.UINT32 ProtobufInput.PrematureEofInPackedField.UINT64 ProtobufInput.PrematureEofInsideKnownRepeatedValue.MESSAGE -TimestampProtoInputTooLarge.JsonOutput -TimestampProtoInputTooSmall.JsonOutput diff --git a/conformance/failure_list_csharp.txt b/conformance/failure_list_csharp.txt index a46cee47..e7de4b96 100644 --- a/conformance/failure_list_csharp.txt +++ b/conformance/failure_list_csharp.txt @@ -1,16 +1,11 @@ -JsonInput.AnyWithValueForInteger.JsonOutput -JsonInput.AnyWithValueForJsonObject.JsonOutput JsonInput.FieldNameInLowerCamelCase.Validator JsonInput.FieldNameInSnakeCase.JsonOutput JsonInput.FieldNameInSnakeCase.ProtobufOutput JsonInput.FieldNameWithMixedCases.JsonOutput JsonInput.FieldNameWithMixedCases.ProtobufOutput JsonInput.FieldNameWithMixedCases.Validator -JsonInput.Int32FieldMinFloatValue.JsonOutput -JsonInput.Int32FieldMinValue.JsonOutput JsonInput.Int64FieldMaxValueNotQuoted.JsonOutput JsonInput.Int64FieldMaxValueNotQuoted.ProtobufOutput JsonInput.OriginalProtoFieldName.JsonOutput -JsonInput.StringFieldSurrogatePair.JsonOutput JsonInput.Uint64FieldMaxValueNotQuoted.JsonOutput JsonInput.Uint64FieldMaxValueNotQuoted.ProtobufOutput diff --git a/conformance/failure_list_java.txt b/conformance/failure_list_java.txt index 552c0cc9..99232ad2 100644 --- a/conformance/failure_list_java.txt +++ b/conformance/failure_list_java.txt @@ -8,8 +8,6 @@ FieldMaskNumbersDontRoundTrip.JsonOutput FieldMaskPathsDontRoundTrip.JsonOutput FieldMaskTooManyUnderscore.JsonOutput JsonInput.AnyWithFieldMask.ProtobufOutput -JsonInput.AnyWithValueForInteger.JsonOutput -JsonInput.AnyWithValueForJsonObject.JsonOutput JsonInput.BoolFieldAllCapitalFalse JsonInput.BoolFieldAllCapitalTrue JsonInput.BoolFieldCamelCaseFalse @@ -30,8 +28,6 @@ JsonInput.FloatFieldInfinityNotQuoted JsonInput.FloatFieldNanNotQuoted JsonInput.FloatFieldNegativeInfinityNotQuoted JsonInput.Int32FieldLeadingZero -JsonInput.Int32FieldMinFloatValue.JsonOutput -JsonInput.Int32FieldMinValue.JsonOutput JsonInput.Int32FieldNegativeWithLeadingZero JsonInput.Int32FieldPlusSign JsonInput.Int32MapFieldKeyNotQuoted diff --git a/conformance/failure_list_python.txt b/conformance/failure_list_python.txt index d2e52637..550a043f 100644 --- a/conformance/failure_list_python.txt +++ b/conformance/failure_list_python.txt @@ -3,26 +3,7 @@ DurationProtoInputTooSmall.JsonOutput FieldMaskNumbersDontRoundTrip.JsonOutput FieldMaskPathsDontRoundTrip.JsonOutput FieldMaskTooManyUnderscore.JsonOutput -JsonInput.Any.JsonOutput -JsonInput.Any.ProtobufOutput -JsonInput.AnyNested.JsonOutput -JsonInput.AnyNested.ProtobufOutput -JsonInput.AnyUnorderedTypeTag.JsonOutput -JsonInput.AnyUnorderedTypeTag.ProtobufOutput -JsonInput.AnyWithDuration.JsonOutput -JsonInput.AnyWithDuration.ProtobufOutput -JsonInput.AnyWithFieldMask.JsonOutput JsonInput.AnyWithFieldMask.ProtobufOutput -JsonInput.AnyWithInt32ValueWrapper.JsonOutput -JsonInput.AnyWithInt32ValueWrapper.ProtobufOutput -JsonInput.AnyWithStruct.JsonOutput -JsonInput.AnyWithStruct.ProtobufOutput -JsonInput.AnyWithTimestamp.JsonOutput -JsonInput.AnyWithTimestamp.ProtobufOutput -JsonInput.AnyWithValueForInteger.JsonOutput -JsonInput.AnyWithValueForInteger.ProtobufOutput -JsonInput.AnyWithValueForJsonObject.JsonOutput -JsonInput.AnyWithValueForJsonObject.ProtobufOutput JsonInput.BytesFieldInvalidBase64Characters JsonInput.DoubleFieldInfinityNotQuoted JsonInput.DoubleFieldNanNotQuoted @@ -54,32 +35,13 @@ JsonInput.Int32FieldMaxFloatValue.JsonOutput JsonInput.Int32FieldMaxFloatValue.ProtobufOutput JsonInput.Int32FieldMinFloatValue.JsonOutput JsonInput.Int32FieldMinFloatValue.ProtobufOutput -JsonInput.Int32FieldMinValue.JsonOutput JsonInput.OriginalProtoFieldName.JsonOutput JsonInput.OriginalProtoFieldName.ProtobufOutput -JsonInput.RepeatedFieldMessageElementIsNull -JsonInput.RepeatedFieldPrimitiveElementIsNull JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool -JsonInput.StringFieldSurrogatePair.JsonOutput -JsonInput.StringFieldUnpairedLowSurrogate -JsonInput.Struct.JsonOutput -JsonInput.Struct.ProtobufOutput JsonInput.TimestampJsonInputLowercaseT JsonInput.Uint32FieldMaxFloatValue.JsonOutput JsonInput.Uint32FieldMaxFloatValue.ProtobufOutput -JsonInput.ValueAcceptBool.JsonOutput -JsonInput.ValueAcceptBool.ProtobufOutput -JsonInput.ValueAcceptFloat.JsonOutput -JsonInput.ValueAcceptFloat.ProtobufOutput -JsonInput.ValueAcceptInteger.JsonOutput -JsonInput.ValueAcceptInteger.ProtobufOutput -JsonInput.ValueAcceptList.JsonOutput -JsonInput.ValueAcceptList.ProtobufOutput JsonInput.ValueAcceptNull.JsonOutput JsonInput.ValueAcceptNull.ProtobufOutput -JsonInput.ValueAcceptObject.JsonOutput -JsonInput.ValueAcceptObject.ProtobufOutput -JsonInput.ValueAcceptString.JsonOutput -JsonInput.ValueAcceptString.ProtobufOutput TimestampProtoInputTooLarge.JsonOutput TimestampProtoInputTooSmall.JsonOutput diff --git a/conformance/failure_list_python_cpp.txt b/conformance/failure_list_python_cpp.txt index 7b5e45f9..1eb916ab 100644 --- a/conformance/failure_list_python_cpp.txt +++ b/conformance/failure_list_python_cpp.txt @@ -12,26 +12,7 @@ DurationProtoInputTooSmall.JsonOutput FieldMaskNumbersDontRoundTrip.JsonOutput FieldMaskPathsDontRoundTrip.JsonOutput FieldMaskTooManyUnderscore.JsonOutput -JsonInput.Any.JsonOutput -JsonInput.Any.ProtobufOutput -JsonInput.AnyNested.JsonOutput -JsonInput.AnyNested.ProtobufOutput -JsonInput.AnyUnorderedTypeTag.JsonOutput -JsonInput.AnyUnorderedTypeTag.ProtobufOutput -JsonInput.AnyWithDuration.JsonOutput -JsonInput.AnyWithDuration.ProtobufOutput -JsonInput.AnyWithFieldMask.JsonOutput JsonInput.AnyWithFieldMask.ProtobufOutput -JsonInput.AnyWithInt32ValueWrapper.JsonOutput -JsonInput.AnyWithInt32ValueWrapper.ProtobufOutput -JsonInput.AnyWithStruct.JsonOutput -JsonInput.AnyWithStruct.ProtobufOutput -JsonInput.AnyWithTimestamp.JsonOutput -JsonInput.AnyWithTimestamp.ProtobufOutput -JsonInput.AnyWithValueForInteger.JsonOutput -JsonInput.AnyWithValueForInteger.ProtobufOutput -JsonInput.AnyWithValueForJsonObject.JsonOutput -JsonInput.AnyWithValueForJsonObject.ProtobufOutput JsonInput.BytesFieldInvalidBase64Characters JsonInput.DoubleFieldInfinityNotQuoted JsonInput.DoubleFieldNanNotQuoted @@ -63,33 +44,14 @@ JsonInput.Int32FieldMaxFloatValue.JsonOutput JsonInput.Int32FieldMaxFloatValue.ProtobufOutput JsonInput.Int32FieldMinFloatValue.JsonOutput JsonInput.Int32FieldMinFloatValue.ProtobufOutput -JsonInput.Int32FieldMinValue.JsonOutput JsonInput.OriginalProtoFieldName.JsonOutput JsonInput.OriginalProtoFieldName.ProtobufOutput -JsonInput.RepeatedFieldMessageElementIsNull -JsonInput.RepeatedFieldPrimitiveElementIsNull JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool -JsonInput.StringFieldSurrogatePair.JsonOutput -JsonInput.StringFieldUnpairedLowSurrogate -JsonInput.Struct.JsonOutput -JsonInput.Struct.ProtobufOutput JsonInput.TimestampJsonInputLowercaseT JsonInput.Uint32FieldMaxFloatValue.JsonOutput JsonInput.Uint32FieldMaxFloatValue.ProtobufOutput -JsonInput.ValueAcceptBool.JsonOutput -JsonInput.ValueAcceptBool.ProtobufOutput -JsonInput.ValueAcceptFloat.JsonOutput -JsonInput.ValueAcceptFloat.ProtobufOutput -JsonInput.ValueAcceptInteger.JsonOutput -JsonInput.ValueAcceptInteger.ProtobufOutput -JsonInput.ValueAcceptList.JsonOutput -JsonInput.ValueAcceptList.ProtobufOutput JsonInput.ValueAcceptNull.JsonOutput JsonInput.ValueAcceptNull.ProtobufOutput -JsonInput.ValueAcceptObject.JsonOutput -JsonInput.ValueAcceptObject.ProtobufOutput -JsonInput.ValueAcceptString.JsonOutput -JsonInput.ValueAcceptString.ProtobufOutput ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE ProtobufInput.PrematureEofInPackedField.BOOL diff --git a/conformance/update_failure_list.py b/conformance/update_failure_list.py new file mode 100755 index 00000000..69f210e3 --- /dev/null +++ b/conformance/update_failure_list.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Script to update a failure list file to add/remove failures. + +This is sort of like comm(1), except it recognizes comments and ignores them. +""" + +import argparse +import fileinput + +parser = argparse.ArgumentParser( + description='Adds/removes failures from the failure list.') +parser.add_argument('filename', type=str, help='failure list file to update') +parser.add_argument('--add', dest='add_list', action='append') +parser.add_argument('--remove', dest='remove_list', action='append') + +args = parser.parse_args() + +add_set = set() +remove_set = set() + +for add_file in (args.add_list or []): + with open(add_file) as f: + for line in f: + add_set.add(line) + +for remove_file in (args.remove_list or []): + with open(remove_file) as f: + for line in f: + if line in add_set: + raise "Asked to both add and remove test: " + line + remove_set.add(line.strip()) + +add_list = sorted(add_set, reverse=True) + +existing_list = file(args.filename).read() + +with open(args.filename, "w") as f: + for line in existing_list.splitlines(True): + test = line.split("#")[0].strip() + while len(add_list) > 0 and test > add_list[-1]: + f.write(add_list.pop()) + if test not in remove_set: + f.write(line) diff --git a/docs/third_party.md b/docs/third_party.md index eaccf782..021cc564 100644 --- a/docs/third_party.md +++ b/docs/third_party.md @@ -146,3 +146,4 @@ There are miscellaneous other things you may find useful as a Protocol Buffers d * [ProtoBuf with Java EE7 Expression Language 3.0; pure Java ProtoBuf Parser and Builder.](https://github.com/protobufel/protobuf-el) * [Notepad++ Syntax Highlighting for .proto files](https://github.com/chai2010/notepadplus-protobuf) * [Linter for .proto files](https://github.com/ckaznocha/protoc-gen-lint) +* [Protocol Buffers Dynamic Schema - create protobuf schemas programmatically (Java)] (https://github.com/os72/protobuf-dynamic) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt new file mode 100644 index 00000000..c9d46885 --- /dev/null +++ b/examples/CMakeLists.txt @@ -0,0 +1,63 @@ +# Minimum CMake required
+cmake_minimum_required(VERSION 2.8.12)
+
+# Project
+project(protobuf-examples)
+
+# Find required protobuf package
+find_package(protobuf CONFIG REQUIRED)
+
+if(protobuf_VERBOSE)
+ message(STATUS "Using Protocol Buffers ${Protobuf_VERSION}")
+endif()
+
+set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
+
+# http://www.cmake.org/Wiki/CMake_FAQ#How_can_I_build_my_MSVC_application_with_a_static_runtime.3F
+if(MSVC AND protobuf_MSVC_STATIC_RUNTIME)
+ foreach(flag_var
+ CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
+ CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
+ if(${flag_var} MATCHES "/MD")
+ string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}")
+ endif(${flag_var} MATCHES "/MD")
+ endforeach()
+endif()
+
+foreach(example add_person list_people)
+ set(${example}_SRCS ${example}.cc)
+ set(${example}_PROTOS addressbook.proto)
+
+ #Code Generation
+ if(protobuf_MODULE_COMPATIBLE) #Legacy Support
+ protobuf_generate_cpp(${example}_PROTO_SRCS ${example}_PROTO_HDRS ${${example}_PROTOS})
+ list(APPEND ${example}_SRCS ${${example}_PROTO_SRCS} ${${example}_PROTO_HDRS})
+ else()
+
+ foreach(proto_file ${${example}_PROTOS})
+ get_filename_component(proto_file_abs ${proto_file} ABSOLUTE)
+ get_filename_component(basename ${proto_file} NAME_WE)
+ set(generated_files ${basename}.pb.cc ${basename}.pb.h)
+ list(APPEND ${example}_SRCS ${generated_files})
+
+ add_custom_command(
+ OUTPUT ${generated_files}
+ COMMAND protobuf::protoc
+ ARGS --cpp_out ${CMAKE_CURRENT_BINARY_DIR} -I ${CMAKE_CURRENT_SOURCE_DIR} ${proto_file_abs}
+ COMMENT "Generating ${generated_files} from ${proto_file}"
+ VERBATIM
+ )
+ endforeach()
+ endif()
+
+ #Executable setup
+ set(executable_name ${example}_cpp)
+ add_executable(${executable_name} ${${example}_SRCS} ${${example}_PROTOS})
+ if(protobuf_MODULE_COMPATIBLE) #Legacy mode
+ target_include_directories(${executable_name} PUBLIC ${PROTOBUF_INCLUDE_DIRS})
+ target_link_libraries(${executable_name} ${PROTOBUF_LIBRARIES})
+ else()
+ target_link_libraries(${executable_name} protobuf::libprotobuf)
+ endif()
+
+endforeach()
diff --git a/objectivec/GPBCodedInputStream.h b/objectivec/GPBCodedInputStream.h index 44f69906..df9d97ba 100644 --- a/objectivec/GPBCodedInputStream.h +++ b/objectivec/GPBCodedInputStream.h @@ -35,6 +35,39 @@ NS_ASSUME_NONNULL_BEGIN +CF_EXTERN_C_BEGIN + +/// GPBCodedInputStream exception name. Exceptions raised from +/// GPBCodedInputStream contain an underlying error in the userInfo dictionary +/// under the GPBCodedInputStreamUnderlyingErrorKey key. +extern NSString *const GPBCodedInputStreamException; + +/// The key under which the underlying NSError from the exception is stored. +extern NSString *const GPBCodedInputStreamUnderlyingErrorKey; + +/// NSError domain used for GPBCodedInputStream errors. +extern NSString *const GPBCodedInputStreamErrorDomain; + +/// Error code for NSError with GPBCodedInputStreamErrorDomain. +typedef NS_ENUM(NSInteger, GPBCodedInputStreamErrorCode) { + /// The size does not fit in the remaining bytes to be read. + GPBCodedInputStreamErrorInvalidSize = -100, + /// Attempted to read beyond the subsection limit. + GPBCodedInputStreamErrorSubsectionLimitReached = -101, + /// The requested subsection limit is invalid. + GPBCodedInputStreamErrorInvalidSubsectionLimit = -102, + /// Invalid tag read. + GPBCodedInputStreamErrorInvalidTag = -103, + /// Invalid UTF-8 character in a string. + GPBCodedInputStreamErrorInvalidUTF8 = -104, + /// Invalid VarInt read. + GPBCodedInputStreamErrorInvalidVarInt = -105, + /// The maximum recursion depth of messages was exceeded. + GPBCodedInputStreamErrorRecursionDepthExceeded = -106, +}; + +CF_EXTERN_C_END + /// Reads and decodes protocol message fields. /// /// The common uses of protocol buffers shouldn't need to use this class. diff --git a/objectivec/GPBCodedInputStream.m b/objectivec/GPBCodedInputStream.m index 40227178..acba7156 100644 --- a/objectivec/GPBCodedInputStream.m +++ b/objectivec/GPBCodedInputStream.m @@ -36,17 +36,42 @@ #import "GPBUtilities_PackagePrivate.h" #import "GPBWireFormat.h" +NSString *const GPBCodedInputStreamException = + GPBNSStringifySymbol(GPBCodedInputStreamException); + +NSString *const GPBCodedInputStreamUnderlyingErrorKey = + GPBNSStringifySymbol(GPBCodedInputStreamUnderlyingErrorKey); + +NSString *const GPBCodedInputStreamErrorDomain = + GPBNSStringifySymbol(GPBCodedInputStreamErrorDomain); + static const NSUInteger kDefaultRecursionLimit = 64; +static void RaiseException(NSInteger code, NSString *reason) { + NSDictionary *errorInfo = nil; + if ([reason length]) { + errorInfo = @{ GPBErrorReasonKey: reason }; + } + NSError *error = [NSError errorWithDomain:GPBCodedInputStreamErrorDomain + code:code + userInfo:errorInfo]; + + NSDictionary *exceptionInfo = + @{ GPBCodedInputStreamUnderlyingErrorKey: error }; + [[[NSException alloc] initWithName:GPBCodedInputStreamException + reason:reason + userInfo:exceptionInfo] raise]; +} + static void CheckSize(GPBCodedInputStreamState *state, size_t size) { size_t newSize = state->bufferPos + size; if (newSize > state->bufferSize) { - [NSException raise:NSParseErrorException format:@""]; + RaiseException(GPBCodedInputStreamErrorInvalidSize, nil); } if (newSize > state->currentLimit) { // Fast forward to end of currentLimit; state->bufferPos = state->currentLimit; - [NSException raise:NSParseErrorException format:@""]; + RaiseException(GPBCodedInputStreamErrorSubsectionLimitReached, nil); } } @@ -95,8 +120,8 @@ static int32_t ReadRawVarint32(GPBCodedInputStreamState *state) { return result; } } - [NSException raise:NSParseErrorException - format:@"Unable to read varint32"]; + RaiseException(GPBCodedInputStreamErrorInvalidVarInt, + @"Invalid VarInt32"); } } } @@ -115,7 +140,7 @@ static int64_t ReadRawVarint64(GPBCodedInputStreamState *state) { } shift += 7; } - [NSException raise:NSParseErrorException format:@"Unable to read varint64"]; + RaiseException(GPBCodedInputStreamErrorInvalidVarInt, @"Invalid VarInt64"); return 0; } @@ -202,8 +227,7 @@ int32_t GPBCodedInputStreamReadTag(GPBCodedInputStreamState *state) { state->lastTag = ReadRawVarint32(state); if (state->lastTag == 0) { // If we actually read zero, that's not a valid tag. - [NSException raise:NSParseErrorException - format:@"Invalid last tag %d", state->lastTag]; + RaiseException(GPBCodedInputStreamErrorInvalidTag, @"Last tag can't be 0"); } return state->lastTag; } @@ -226,8 +250,7 @@ NSString *GPBCodedInputStreamReadRetainedString( NSLog(@"UTF-8 failure, is some field type 'string' when it should be " @"'bytes'?"); #endif - [NSException raise:NSParseErrorException - format:@"Invalid UTF-8 for a 'string'"]; + RaiseException(GPBCodedInputStreamErrorInvalidUTF8, nil); } } return result; @@ -262,8 +285,7 @@ size_t GPBCodedInputStreamPushLimit(GPBCodedInputStreamState *state, byteLimit += state->bufferPos; size_t oldLimit = state->currentLimit; if (byteLimit > oldLimit) { - [NSException raise:NSInvalidArgumentException - format:@"byteLimit > oldLimit: %tu > %tu", byteLimit, oldLimit]; + RaiseException(GPBCodedInputStreamErrorInvalidSubsectionLimit, nil); } state->currentLimit = byteLimit; return oldLimit; @@ -286,8 +308,7 @@ BOOL GPBCodedInputStreamIsAtEnd(GPBCodedInputStreamState *state) { void GPBCodedInputStreamCheckLastTagWas(GPBCodedInputStreamState *state, int32_t value) { if (state->lastTag != value) { - [NSException raise:NSParseErrorException - format:@"Last tag: %d should be %d", state->lastTag, value]; + RaiseException(GPBCodedInputStreamErrorInvalidTag, @"Unexpected tag read"); } } @@ -353,7 +374,7 @@ void GPBCodedInputStreamCheckLastTagWas(GPBCodedInputStreamState *state, SkipRawData(&state_, sizeof(int32_t)); return YES; } - [NSException raise:NSParseErrorException format:@"Invalid tag %d", tag]; + RaiseException(GPBCodedInputStreamErrorInvalidTag, nil); return NO; } @@ -414,9 +435,7 @@ void GPBCodedInputStreamCheckLastTagWas(GPBCodedInputStreamState *state, message:(GPBMessage *)message extensionRegistry:(GPBExtensionRegistry *)extensionRegistry { if (state_.recursionDepth >= kDefaultRecursionLimit) { - [NSException raise:NSParseErrorException - format:@"recursionDepth(%tu) >= %tu", state_.recursionDepth, - kDefaultRecursionLimit]; + RaiseException(GPBCodedInputStreamErrorRecursionDepthExceeded, nil); } ++state_.recursionDepth; [message mergeFromCodedInputStream:self extensionRegistry:extensionRegistry]; @@ -428,9 +447,7 @@ void GPBCodedInputStreamCheckLastTagWas(GPBCodedInputStreamState *state, - (void)readUnknownGroup:(int32_t)fieldNumber message:(GPBUnknownFieldSet *)message { if (state_.recursionDepth >= kDefaultRecursionLimit) { - [NSException raise:NSParseErrorException - format:@"recursionDepth(%tu) >= %tu", state_.recursionDepth, - kDefaultRecursionLimit]; + RaiseException(GPBCodedInputStreamErrorRecursionDepthExceeded, nil); } ++state_.recursionDepth; [message mergeFromCodedInputStream:self]; @@ -443,9 +460,7 @@ void GPBCodedInputStreamCheckLastTagWas(GPBCodedInputStreamState *state, extensionRegistry:(GPBExtensionRegistry *)extensionRegistry { int32_t length = ReadRawVarint32(&state_); if (state_.recursionDepth >= kDefaultRecursionLimit) { - [NSException raise:NSParseErrorException - format:@"recursionDepth(%tu) >= %tu", state_.recursionDepth, - kDefaultRecursionLimit]; + RaiseException(GPBCodedInputStreamErrorRecursionDepthExceeded, nil); } size_t oldLimit = GPBCodedInputStreamPushLimit(&state_, length); ++state_.recursionDepth; @@ -461,9 +476,7 @@ void GPBCodedInputStreamCheckLastTagWas(GPBCodedInputStreamState *state, parentMessage:(GPBMessage *)parentMessage { int32_t length = ReadRawVarint32(&state_); if (state_.recursionDepth >= kDefaultRecursionLimit) { - [NSException raise:NSParseErrorException - format:@"recursionDepth(%tu) >= %tu", state_.recursionDepth, - kDefaultRecursionLimit]; + RaiseException(GPBCodedInputStreamErrorRecursionDepthExceeded, nil); } size_t oldLimit = GPBCodedInputStreamPushLimit(&state_, length); ++state_.recursionDepth; diff --git a/objectivec/GPBMessage.h b/objectivec/GPBMessage.h index 2249829f..b3b07793 100644 --- a/objectivec/GPBMessage.h +++ b/objectivec/GPBMessage.h @@ -49,12 +49,15 @@ extern NSString *const GPBMessageErrorDomain; /// Error code for NSError with GPBMessageErrorDomain. typedef NS_ENUM(NSInteger, GPBMessageErrorCode) { - /// The data being parsed is bad and a message can not be created from it. - GPBMessageErrorCodeMalformedData = -100, + /// Uncategorized error. + GPBMessageErrorCodeOther = -100, /// A message can't be serialized because it is missing required fields. GPBMessageErrorCodeMissingRequiredField = -101, }; +/// Key under which the error's reason is stored inside the userInfo dictionary. +extern NSString *const GPBErrorReasonKey; + CF_EXTERN_C_END /// Base class for all of the generated message classes. @@ -86,6 +89,9 @@ CF_EXTERN_C_END /// @note In DEBUG builds, the parsed message is checked to be sure all required /// fields were provided, and the parse will fail if some are missing. /// +/// @note The errors returned are likely coming from the domain and codes listed +/// at the top of this file and GPBCodedInputStream.h. +/// /// @param data The data to parse. /// @param errorPtr An optional error pointer to fill in with a failure reason if /// the data can not be parsed. @@ -101,6 +107,9 @@ CF_EXTERN_C_END /// @note In DEBUG builds, the parsed message is checked to be sure all required /// fields were provided, and the parse will fail if some are missing. /// +/// @note The errors returned are likely coming from the domain and codes listed +/// at the top of this file and GPBCodedInputStream.h. +/// /// @param data The data to parse. /// @param extensionRegistry The extension registry to use to look up extensions. /// @param errorPtr An optional error pointer to fill in with a failure @@ -119,6 +128,9 @@ CF_EXTERN_C_END /// @note In DEBUG builds, the parsed message is checked to be sure all required /// fields were provided, and the parse will fail if some are missing. /// +/// @note The errors returned are likely coming from the domain and codes listed +/// at the top of this file and GPBCodedInputStream.h. +/// /// @param input The stream to read data from. /// @param extensionRegistry The extension registry to use to look up extensions. /// @param errorPtr An optional error pointer to fill in with a failure @@ -139,6 +151,9 @@ CF_EXTERN_C_END /// the required fields are set. So this method can be used to reload /// messages that may not be complete. /// +/// @note The errors returned are likely coming from the domain and codes listed +/// at the top of this file and GPBCodedInputStream.h. +/// /// @param input The stream to read data from. /// @param extensionRegistry The extension registry to use to look up extensions. /// @param errorPtr An optional error pointer to fill in with a failure @@ -158,6 +173,9 @@ CF_EXTERN_C_END /// @note In DEBUG builds, the parsed message is checked to be sure all required /// fields were provided, and the parse will fail if some are missing. /// +/// @note The errors returned are likely coming from the domain and codes listed +/// at the top of this file and GPBCodedInputStream.h. +/// /// @param data The data to parse. /// @param errorPtr An optional error pointer to fill in with a failure reason if /// the data can not be parsed. @@ -171,6 +189,9 @@ CF_EXTERN_C_END /// @note In DEBUG builds, the parsed message is checked to be sure all required /// fields were provided, and the parse will fail if some are missing. /// +/// @note The errors returned are likely coming from the domain and codes listed +/// at the top of this file and GPBCodedInputStream.h. +/// /// @param data The data to parse. /// @param extensionRegistry The extension registry to use to look up extensions. /// @param errorPtr An optional error pointer to fill in with a failure @@ -188,6 +209,9 @@ CF_EXTERN_C_END /// the required fields are set. So this method can be used to reload /// messages that may not be complete. /// +/// @note The errors returned are likely coming from the domain and codes listed +/// at the top of this file and GPBCodedInputStream.h. +/// /// @param input The stream to read data from. /// @param extensionRegistry The extension registry to use to look up extensions. /// @param errorPtr An optional error pointer to fill in with a failure diff --git a/objectivec/GPBMessage.m b/objectivec/GPBMessage.m index f1cb3927..919fb931 100644 --- a/objectivec/GPBMessage.m +++ b/objectivec/GPBMessage.m @@ -53,6 +53,8 @@ NSString *const GPBMessageErrorDomain = GPBNSStringifySymbol(GPBMessageErrorDomain); +NSString *const GPBErrorReasonKey = @"Reason"; + static NSString *const kGPBDataCoderKey = @"GPBData"; // @@ -96,20 +98,35 @@ static NSMutableDictionary *CloneExtensionMap(NSDictionary *extensionMap, NSZone *zone) __attribute__((ns_returns_retained)); +#ifdef DEBUG static NSError *MessageError(NSInteger code, NSDictionary *userInfo) { return [NSError errorWithDomain:GPBMessageErrorDomain code:code userInfo:userInfo]; } +#endif -static NSError *MessageErrorWithReason(NSInteger code, NSString *reason) { - NSDictionary *userInfo = nil; - if ([reason length]) { - userInfo = @{ @"Reason" : reason }; +static NSError *ErrorFromException(NSException *exception) { + NSError *error = nil; + + if ([exception.name isEqual:GPBCodedInputStreamException]) { + NSDictionary *exceptionInfo = exception.userInfo; + error = exceptionInfo[GPBCodedInputStreamUnderlyingErrorKey]; } - return MessageError(code, userInfo); -} + if (!error) { + NSString *reason = exception.reason; + NSDictionary *userInfo = nil; + if ([reason length]) { + userInfo = @{ GPBErrorReasonKey : reason }; + } + + error = [NSError errorWithDomain:GPBMessageErrorDomain + code:GPBMessageErrorCodeOther + userInfo:userInfo]; + } + return error; +} static void CheckExtension(GPBMessage *self, GPBExtensionDescriptor *extension) { @@ -817,8 +834,7 @@ static GPBUnknownFieldSet *GetOrMakeUnknownFields(GPBMessage *self) { [self release]; self = nil; if (errorPtr) { - *errorPtr = MessageErrorWithReason(GPBMessageErrorCodeMalformedData, - exception.reason); + *errorPtr = ErrorFromException(exception); } } #ifdef DEBUG @@ -849,8 +865,7 @@ static GPBUnknownFieldSet *GetOrMakeUnknownFields(GPBMessage *self) { [self release]; self = nil; if (errorPtr) { - *errorPtr = MessageErrorWithReason(GPBMessageErrorCodeMalformedData, - exception.reason); + *errorPtr = ErrorFromException(exception); } } #ifdef DEBUG @@ -1923,8 +1938,7 @@ static GPBUnknownFieldSet *GetOrMakeUnknownFields(GPBMessage *self) { @catch (NSException *exception) { message = nil; if (errorPtr) { - *errorPtr = MessageErrorWithReason(GPBMessageErrorCodeMalformedData, - exception.reason); + *errorPtr = ErrorFromException(exception); } } #ifdef DEBUG diff --git a/objectivec/Tests/GPBMessageTests+Serialization.m b/objectivec/Tests/GPBMessageTests+Serialization.m index 0d811a96..fad9773a 100644 --- a/objectivec/Tests/GPBMessageTests+Serialization.m +++ b/objectivec/Tests/GPBMessageTests+Serialization.m @@ -881,6 +881,103 @@ static NSData *DataFromCStr(const char *str) { XCTAssertEqualObjects(extsParse, extsOrig); } +- (void)testErrorSubsectionInvalidLimit { + NSData *data = DataFromCStr( + "\x0A\x08\x0A\x07\x12\x04\x72\x02\x4B\x50\x12\x04\x72\x02\x4B\x50"); + NSError *error = nil; + NestedTestAllTypes *msg = [NestedTestAllTypes parseFromData:data + error:&error]; + XCTAssertNil(msg); + XCTAssertNotNil(error); + XCTAssertEqualObjects(error.domain, GPBCodedInputStreamErrorDomain); + XCTAssertEqual(error.code, GPBCodedInputStreamErrorInvalidSubsectionLimit); +} + +- (void)testErrorSubsectionLimitReached { + NSData *data = DataFromCStr("\x0A\x06\x12\x03\x72\x02\x4B\x50"); + NSError *error = nil; + NestedTestAllTypes *msg = [NestedTestAllTypes parseFromData:data + error:&error]; + XCTAssertNil(msg); + XCTAssertNotNil(error); + XCTAssertEqualObjects(error.domain, GPBCodedInputStreamErrorDomain); + XCTAssertEqual(error.code, GPBCodedInputStreamErrorSubsectionLimitReached); +} + +- (void)testErrorInvalidVarint { + NSData *data = DataFromCStr("\x72\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"); + NSError *error = nil; + TestAllTypes *msg = [TestAllTypes parseFromData:data error:&error]; + XCTAssertNil(msg); + XCTAssertNotNil(error); + XCTAssertEqualObjects(error.domain, GPBCodedInputStreamErrorDomain); + XCTAssertEqual(error.code, GPBCodedInputStreamErrorInvalidVarInt); +} + +- (void)testErrorInvalidUTF8 { + NSData *data = DataFromCStr("\x72\x04\xF4\xFF\xFF\xFF"); + NSError *error = nil; + TestAllTypes *msg = [TestAllTypes parseFromData:data error:&error]; + XCTAssertNil(msg); + XCTAssertNotNil(error); + XCTAssertEqualObjects(error.domain, GPBCodedInputStreamErrorDomain); + XCTAssertEqual(error.code, GPBCodedInputStreamErrorInvalidUTF8); +} + +- (void)testErrorInvalidSize { + NSData *data = DataFromCStr("\x72\x03\x4B\x50"); + NSError *error = nil; + NestedTestAllTypes *msg = [NestedTestAllTypes parseFromData:data + error:&error]; + XCTAssertNil(msg); + XCTAssertNotNil(error); + XCTAssertEqualObjects(error.domain, GPBCodedInputStreamErrorDomain); + XCTAssertEqual(error.code, GPBCodedInputStreamErrorInvalidSize); +} + +- (void)testErrorInvalidTag { + NSData *data = DataFromCStr("\x0F"); + NSError *error = nil; + NestedTestAllTypes *msg = [NestedTestAllTypes parseFromData:data + error:&error]; + XCTAssertNil(msg); + XCTAssertNotNil(error); + XCTAssertEqualObjects(error.domain, GPBCodedInputStreamErrorDomain); + XCTAssertEqual(error.code, GPBCodedInputStreamErrorInvalidTag); +} + +- (void)testErrorRecursionDepthReached { + NSData *data = DataFromCStr( + "\x0A\x86\x01\x0A\x83\x01\x0A\x80\x01\x0A\x7E\x0A\x7C\x0A\x7A\x0A\x78" + "\x0A\x76\x0A\x74\x0A\x72\x0A\x70\x0A\x6E\x0A\x6C\x0A\x6A\x0A\x68" + "\x0A\x66\x0A\x64\x0A\x62\x0A\x60\x0A\x5E\x0A\x5C\x0A\x5A\x0A\x58" + "\x0A\x56\x0A\x54\x0A\x52\x0A\x50\x0A\x4E\x0A\x4C\x0A\x4A\x0A\x48" + "\x0A\x46\x0A\x44\x0A\x42\x0A\x40\x0A\x3E\x0A\x3C\x0A\x3A\x0A\x38" + "\x0A\x36\x0A\x34\x0A\x32\x0A\x30\x0A\x2E\x0A\x2C\x0A\x2A\x0A\x28" + "\x0A\x26\x0A\x24\x0A\x22\x0A\x20\x0A\x1E\x0A\x1C\x0A\x1A\x0A\x18" + "\x0A\x16\x0A\x14\x0A\x12\x0A\x10\x0A\x0E\x0A\x0C\x0A\x0A\x0A\x08" + "\x0A\x06\x12\x04\x72\x02\x4B\x50"); + NSError *error = nil; + NestedTestAllTypes *msg = [NestedTestAllTypes parseFromData:data + error:&error]; + XCTAssertNil(msg); + XCTAssertNotNil(error); + XCTAssertEqualObjects(error.domain, GPBCodedInputStreamErrorDomain); + XCTAssertEqual(error.code, GPBCodedInputStreamErrorRecursionDepthExceeded); +} + +#ifdef DEBUG +- (void)testErrorMissingRequiredField { + NSData *data = DataFromCStr(""); + NSError *error = nil; + TestRequired *msg = [TestRequired parseFromData:data error:&error]; + XCTAssertNil(msg); + XCTAssertNotNil(error); + XCTAssertEqualObjects(error.domain, GPBMessageErrorDomain); + XCTAssertEqual(error.code, GPBMessageErrorCodeMissingRequiredField); +} +#endif + #pragma mark - Subset from from map_tests.cc // TEST(GeneratedMapFieldTest, StandardWireFormat) @@ -989,8 +1086,8 @@ static NSData *DataFromCStr(const char *str) { TestMap *msg = [TestMap parseFromData:data error:&error]; XCTAssertNil(msg); XCTAssertNotNil(error); - XCTAssertEqualObjects(error.domain, GPBMessageErrorDomain); - XCTAssertEqual(error.code, GPBMessageErrorCodeMalformedData); + XCTAssertEqualObjects(error.domain, GPBCodedInputStreamErrorDomain); + XCTAssertEqual(error.code, GPBCodedInputStreamErrorInvalidSubsectionLimit); } // TEST(GeneratedMapFieldTest, Proto2UnknownEnum) diff --git a/python/google/protobuf/internal/json_format_test.py b/python/google/protobuf/internal/json_format_test.py index eec1f56f..9e32ea47 100644 --- a/python/google/protobuf/internal/json_format_test.py +++ b/python/google/protobuf/internal/json_format_test.py @@ -247,6 +247,27 @@ class JsonFormatTest(JsonFormatBase): parsed_message = json_format_proto3_pb2.TestOneof() self.CheckParseBack(message, parsed_message) + def testSurrogates(self): + # Test correct surrogate handling. + message = json_format_proto3_pb2.TestMessage() + json_format.Parse('{"stringValue": "\\uD83D\\uDE01"}', message) + self.assertEqual(message.string_value, + b'\xF0\x9F\x98\x81'.decode("utf-8", "strict")) + + # TODO: add test that UTF-8 encoded surrogate code points are rejected. + # UTF-8 does not allow them. + + # Error case: unpaired high surrogate. + self.CheckError( + '{"stringValue": "\\uD83D"}', + r'Invalid \\uXXXX escape|Unpaired.*surrogate') + + # Unpaired low surrogate. + self.CheckError( + '{"stringValue": "\\uDE01"}', + r'Invalid \\uXXXX escape|Unpaired.*surrogate') + + def testTimestampMessage(self): message = json_format_proto3_pb2.TestTimestamp() message.value.seconds = 0 diff --git a/python/google/protobuf/json_format.py b/python/google/protobuf/json_format.py index 57aa4077..be6a9b63 100644 --- a/python/google/protobuf/json_format.py +++ b/python/google/protobuf/json_format.py @@ -49,6 +49,7 @@ except ImportError: import base64 import json import math +import re import six import sys @@ -68,6 +69,9 @@ _INFINITY = 'Infinity' _NEG_INFINITY = '-Infinity' _NAN = 'NaN' +_UNPAIRED_SURROGATE_PATTERN = re.compile(six.u( + r'[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]' +)) class Error(Exception): """Top-level module error for json_format.""" @@ -555,6 +559,10 @@ def _ConvertScalarFieldValue(value, field, require_str=False): if field.type == descriptor.FieldDescriptor.TYPE_BYTES: return base64.b64decode(value) else: + # Checking for unpaired surrogates appears to be unreliable, + # depending on the specific Python version, so we check manually. + if _UNPAIRED_SURROGATE_PATTERN.search(value): + raise ParseError('Unpaired surrogate') return value elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM: # Convert an enum value. |