diff options
author | Feng Xiao <xfxyjwf@gmail.com> | 2018-07-17 18:50:04 -0700 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-07-17 18:50:04 -0700 |
commit | d85ffdce419eb3e5b8444e2fed7c9e13f3b9637e (patch) | |
tree | 6224f98ad30e99dc2d896423853892f2142f16ed | |
parent | 22503a01722939385a7584c3b65f1c43dc5b2ef8 (diff) | |
parent | 8356d270a54e18c21f4feac6f5e2b6f1061dc8d5 (diff) | |
download | protobuf-d85ffdce419eb3e5b8444e2fed7c9e13f3b9637e.tar.gz protobuf-d85ffdce419eb3e5b8444e2fed7c9e13f3b9637e.tar.bz2 protobuf-d85ffdce419eb3e5b8444e2fed7c9e13f3b9637e.zip |
Merge pull request #4924 from xfxyjwf/3.6.x
Cherry-pick master build fixes and kokoro changes to 3.6.x
33 files changed, 663 insertions, 506 deletions
@@ -47,7 +47,6 @@ any_test.pb.* map*unittest.pb.* unittest*.pb.* cpp_test*.pb.* -src/google/protobuf/compiler/js/well_known_types_embed.cc src/google/protobuf/util/**/*.pb.cc src/google/protobuf/util/**/*.pb.h @@ -257,24 +257,6 @@ internal_copied_filegroup( # Protocol Buffers Compiler ################################################################################ -cc_binary( - name = "js_embed", - srcs = ["src/google/protobuf/compiler/js/embed.cc"], - visibility = ["//visibility:public"], -) - -genrule( - name = "generate_js_well_known_types_embed", - srcs = [ - "src/google/protobuf/compiler/js/well_known_types/any.js", - "src/google/protobuf/compiler/js/well_known_types/struct.js", - "src/google/protobuf/compiler/js/well_known_types/timestamp.js", - ], - outs = ["src/google/protobuf/compiler/js/well_known_types_embed.cc"], - cmd = "$(location :js_embed) $(SRCS) > $@", - tools = [":js_embed"], -) - cc_library( name = "protoc_lib", srcs = [ @@ -28,6 +28,11 @@ fi set -ex +# The absence of a m4 directory in googletest causes autoreconf to fail when +# building under the CentOS docker image. It's a warning in regular build on +# Ubuntu/gLinux as well. +mkdir -p third_party/googletest/m4 + # TODO(kenton): Remove the ",no-obsolete" part and fix the resulting warnings. autoreconf -f -i -Wall,no-obsolete diff --git a/benchmarks/Makefile.am b/benchmarks/Makefile.am index 564fdab3..3ae14ffb 100644 --- a/benchmarks/Makefile.am +++ b/benchmarks/Makefile.am @@ -109,6 +109,7 @@ cpp_benchmark_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir)/cpp -I$(top_srcdir)/thi # so a direct "make test_cpp" could fail if parallel enough. # See: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html#Recording-Dependencies-manually cpp/cpp_benchmark-cpp_benchmark.$(OBJEXT): $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/third_party/benchmark/src/libbenchmark.a +cpp/benchmark-cpp_benchmark.$(OBJEXT): $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/third_party/benchmark/src/libbenchmark.a nodist_cpp_benchmark_SOURCES = \ $(benchmarks_protoc_outputs) \ $(benchmarks_protoc_outputs_proto2) \ diff --git a/cmake/libprotoc.cmake b/cmake/libprotoc.cmake index ec6958bc..58568bb2 100644 --- a/cmake/libprotoc.cmake +++ b/cmake/libprotoc.cmake @@ -161,17 +161,11 @@ set(libprotoc_headers ${protobuf_source_dir}/src/google/protobuf/compiler/zip_writer.h ) -set(js_well_known_types_sources - ${protobuf_source_dir}/src/google/protobuf/compiler/js/well_known_types/any.js - ${protobuf_source_dir}/src/google/protobuf/compiler/js/well_known_types/struct.js - ${protobuf_source_dir}/src/google/protobuf/compiler/js/well_known_types/timestamp.js -) -add_executable(js_embed ${protobuf_source_dir}/src/google/protobuf/compiler/js/embed.cc) -add_custom_command( - OUTPUT ${protobuf_source_dir}/src/google/protobuf/compiler/js/well_known_types_embed.cc - DEPENDS js_embed ${js_well_known_types_sources} - COMMAND js_embed ${js_well_known_types_sources} > ${protobuf_source_dir}/src/google/protobuf/compiler/js/well_known_types_embed.cc +if (MSVC) +set(libprotoc_rc_files + ${CMAKE_CURRENT_BINARY_DIR}/version.rc ) +endif() add_library(libprotoc ${protobuf_SHARED_OR_STATIC} ${libprotoc_files} ${libprotoc_headers}) diff --git a/conformance/Makefile.am b/conformance/Makefile.am index 765f3588..e51ab80a 100644 --- a/conformance/Makefile.am +++ b/conformance/Makefile.am @@ -279,8 +279,6 @@ $(protoc_outputs): protoc_middleman $(other_language_protoc_outputs): protoc_middleman -BUILT_SOURCES = $(protoc_outputs) $(other_language_protoc_outputs) - CLEANFILES = $(protoc_outputs) protoc_middleman javac_middleman conformance-java javac_middleman_lite conformance-java-lite conformance-csharp conformance-php conformance-php-c $(other_language_protoc_outputs) MAINTAINERCLEANFILES = \ diff --git a/conformance/conformance_php.php b/conformance/conformance_php.php index 19f9a092..85fe3d03 100755 --- a/conformance/conformance_php.php +++ b/conformance/conformance_php.php @@ -6,8 +6,8 @@ require_once("Conformance/ConformanceRequest.php"); require_once("Protobuf_test_messages/Proto3/ForeignMessage.php"); require_once("Protobuf_test_messages/Proto3/ForeignEnum.php"); require_once("Protobuf_test_messages/Proto3/TestAllTypesProto3.php"); -require_once("Protobuf_test_messages/Proto3/TestAllTypesProto3_NestedMessage.php"); -require_once("Protobuf_test_messages/Proto3/TestAllTypesProto3_NestedEnum.php"); +require_once("Protobuf_test_messages/Proto3/TestAllTypesProto3/NestedMessage.php"); +require_once("Protobuf_test_messages/Proto3/TestAllTypesProto3/NestedEnum.php"); require_once("GPBMetadata/Conformance.php"); require_once("GPBMetadata/Google/Protobuf/TestMessagesProto3.php"); diff --git a/examples/list_people_test.go b/examples/list_people_test.go index 87d6ad6e..887997fe 100644 --- a/examples/list_people_test.go +++ b/examples/list_people_test.go @@ -34,7 +34,7 @@ func TestWritePersonWritesPerson(t *testing.T) { func TestListPeopleWritesList(t *testing.T) { buf := new(bytes.Buffer) - in := pb.AddressBook{[]*pb.Person{ + in := pb.AddressBook{People: []*pb.Person { { Name: "John Doe", Id: 101, diff --git a/kokoro/linux/build_and_run_docker.sh b/kokoro/linux/build_and_run_docker.sh index 6a1f327a..f9c60710 100755 --- a/kokoro/linux/build_and_run_docker.sh +++ b/kokoro/linux/build_and_run_docker.sh @@ -35,6 +35,8 @@ echo $git_root docker run \ "$@" \ -e CCACHE_DIR=$CCACHE_DIR \ + -e KOKORO_BUILD_NUMBER=$KOKORO_BUILD_NUMBER \ + -e KOKORO_BUILD_ID=$KOKORO_BUILD_ID \ -e EXTERNAL_GIT_ROOT="/var/local/kokoro/protobuf" \ -e TEST_SET="$TEST_SET" \ -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \ diff --git a/kokoro/linux/cpp_distcheck/build.sh b/kokoro/linux/cpp_distcheck/build.sh index b8b57e35..89b8379e 100755 --- a/kokoro/linux/cpp_distcheck/build.sh +++ b/kokoro/linux/cpp_distcheck/build.sh @@ -5,7 +5,15 @@ # Change to repo root cd $(dirname $0)/../../.. -# Prepare worker environment to run tests -source kokoro/linux/prepare_build_linux_rc - ./tests.sh cpp_distcheck + +# Run tests under release docker image. +DOCKER_IMAGE_NAME=protobuf/protoc_$(sha1sum protoc-artifacts/Dockerfile | cut -f1 -d " ") +docker pull $DOCKER_IMAGE_NAME + +docker run -v $(pwd):/var/local/protobuf --rm $DOCKER_IMAGE_NAME \ + bash -l /var/local/protobuf/tests.sh cpp || FAILED="true" + +if [ "$FAILED" = "true" ]; then + exit 1 +fi diff --git a/kokoro/linux/pull_request_in_docker.sh b/kokoro/linux/pull_request_in_docker.sh index df3636cc..e28ef958 100755 --- a/kokoro/linux/pull_request_in_docker.sh +++ b/kokoro/linux/pull_request_in_docker.sh @@ -5,7 +5,7 @@ WORKSPACE_BASE=`pwd` MY_DIR="$(dirname "$0")" -TEST_SCRIPT=$MY_DIR/../../tests.sh +TEST_SCRIPT=./tests.sh BUILD_DIR=/tmp/protobuf set -e # exit immediately on error @@ -53,7 +53,7 @@ $TIME_CMD $TEST_SCRIPT cpp > >(tee $CPP_STDOUT) 2> >(tee $CPP_STDERR >&2) parallel --results $LOG_OUTPUT_DIR --joblog $OUTPUT_DIR/joblog $TEST_SCRIPT ::: \ $TEST_SET \ - || true # Process test results even if tests fail. + || FAILED="true" # Process test results even if tests fail. cat $OUTPUT_DIR/joblog @@ -67,3 +67,7 @@ TESTOUTPUT_XML_FILE=$COPY_FROM_DOCKER/sponge_log.xml python $MY_DIR/make_test_output.py $OUTPUT_DIR > $TESTOUTPUT_XML_FILE ls -l $TESTOUTPUT_XML_FILE + +if [ "$FAILED" == "true" ]; then + exit 1 +fi diff --git a/kokoro/release/protoc/linux/build.sh b/kokoro/release/protoc/linux/build.sh new file mode 100755 index 00000000..6c7049d7 --- /dev/null +++ b/kokoro/release/protoc/linux/build.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -x + +# Change to repo root. +cd $(dirname $0)/../../../.. + +# Initialize any submodules. +git submodule update --init --recursive + +# Generate the configure script. +./autogen.sh + +# Use docker image to build linux artifacts. +DOCKER_IMAGE_NAME=protobuf/protoc_$(sha1sum protoc-artifacts/Dockerfile | cut -f1 -d " ") +docker pull $DOCKER_IMAGE_NAME + +docker run -v $(pwd):/var/local/protobuf --rm $DOCKER_IMAGE_NAME \ + bash -l /var/local/protobuf/protoc-artifacts/build-protoc.sh \ + linux x86_64 protoc || { + echo "Failed to build protoc for linux + x86_64." + exit 1 +} + +docker run -v $(pwd):/var/local/protobuf --rm $DOCKER_IMAGE_NAME \ + bash -l /var/local/protobuf/protoc-artifacts/build-protoc.sh \ + linux x86_32 protoc || { + echo "Failed to build protoc for linux + x86_32." + exit 1 +} + +# Cross-build for some architectures. +# TODO(xiaofeng): It currently fails with "machine `aarch64' not recognized" +# error. +# sudo apt install -y g++-aarch64-linux-gnu +# protoc-artifacts/build-protoc.sh linux aarch_64 protoc diff --git a/kokoro/release/protoc/linux/release.cfg b/kokoro/release/protoc/linux/release.cfg new file mode 100644 index 00000000..4d8b7b61 --- /dev/null +++ b/kokoro/release/protoc/linux/release.cfg @@ -0,0 +1,7 @@ +build_file: "protobuf/kokoro/release/protoc/linux/build.sh" + +action { + define_artifacts { + regex: "**/protoc.exe" + } +} diff --git a/php/ext/google/protobuf/message.c b/php/ext/google/protobuf/message.c index a40e0f63..06853874 100644 --- a/php/ext/google/protobuf/message.c +++ b/php/ext/google/protobuf/message.c @@ -283,6 +283,7 @@ void build_class_from_descriptor( // ----------------------------------------------------------------------------- void Message_construct(zval* msg, zval* array_wrapper) { + TSRMLS_FETCH(); zend_class_entry* ce = Z_OBJCE_P(msg); MessageHeader* intern = NULL; if (EXPECTED(class_added(ce))) { diff --git a/php/src/Google/Protobuf/Internal/MapField.php b/php/src/Google/Protobuf/Internal/MapField.php index f7d7b710..1b2b0cf0 100644 --- a/php/src/Google/Protobuf/Internal/MapField.php +++ b/php/src/Google/Protobuf/Internal/MapField.php @@ -156,15 +156,22 @@ class MapField implements \ArrayAccess, \IteratorAggregate, \Countable $this->checkKey($this->key_type, $key); switch ($this->value_type) { + case GPBType::SFIXED32: + case GPBType::SINT32: case GPBType::INT32: + case GPBType::ENUM: GPBUtil::checkInt32($value); break; + case GPBType::FIXED32: case GPBType::UINT32: GPBUtil::checkUint32($value); break; + case GPBType::SFIXED64: + case GPBType::SINT64: case GPBType::INT64: GPBUtil::checkInt64($value); break; + case GPBType::FIXED64: case GPBType::UINT64: GPBUtil::checkUint64($value); break; @@ -249,36 +256,24 @@ class MapField implements \ArrayAccess, \IteratorAggregate, \Countable private function checkKey($key_type, &$key) { switch ($key_type) { + case GPBType::SFIXED32: + case GPBType::SINT32: case GPBType::INT32: GPBUtil::checkInt32($key); break; + case GPBType::FIXED32: case GPBType::UINT32: GPBUtil::checkUint32($key); break; + case GPBType::SFIXED64: + case GPBType::SINT64: case GPBType::INT64: GPBUtil::checkInt64($key); break; - case GPBType::UINT64: - GPBUtil::checkUint64($key); - break; case GPBType::FIXED64: + case GPBType::UINT64: GPBUtil::checkUint64($key); break; - case GPBType::FIXED32: - GPBUtil::checkUint32($key); - break; - case GPBType::SFIXED64: - GPBUtil::checkInt64($key); - break; - case GPBType::SFIXED32: - GPBUtil::checkInt32($key); - break; - case GPBType::SINT64: - GPBUtil::checkInt64($key); - break; - case GPBType::SINT32: - GPBUtil::checkInt32($key); - break; case GPBType::BOOL: GPBUtil::checkBool($key); break; diff --git a/php/src/Google/Protobuf/Internal/RepeatedField.php b/php/src/Google/Protobuf/Internal/RepeatedField.php index e0ddef50..e9b92d8d 100644 --- a/php/src/Google/Protobuf/Internal/RepeatedField.php +++ b/php/src/Google/Protobuf/Internal/RepeatedField.php @@ -141,15 +141,22 @@ class RepeatedField implements \ArrayAccess, \IteratorAggregate, \Countable public function offsetSet($offset, $value) { switch ($this->type) { + case GPBType::SFIXED32: + case GPBType::SINT32: case GPBType::INT32: + case GPBType::ENUM: GPBUtil::checkInt32($value); break; + case GPBType::FIXED32: case GPBType::UINT32: GPBUtil::checkUint32($value); break; + case GPBType::SFIXED64: + case GPBType::SINT64: case GPBType::INT64: GPBUtil::checkInt64($value); break; + case GPBType::FIXED64: case GPBType::UINT64: GPBUtil::checkUint64($value); break; @@ -162,6 +169,9 @@ class RepeatedField implements \ArrayAccess, \IteratorAggregate, \Countable case GPBType::BOOL: GPBUtil::checkBool($value); break; + case GPBType::BYTES: + GPBUtil::checkString($value, false); + break; case GPBType::STRING: GPBUtil::checkString($value, true); break; diff --git a/php/tests/compatibility_test.sh b/php/tests/compatibility_test.sh index b5b255ea..b377d85c 100755 --- a/php/tests/compatibility_test.sh +++ b/php/tests/compatibility_test.sh @@ -122,6 +122,8 @@ composer install tests=( array_test.php encode_decode_test.php generated_class_test.php map_field_test.php well_known_test.php ) sed -i.bak '/php_implementation_test.php/d' phpunit.xml sed -i.bak '/generated_phpdoc_test.php/d' phpunit.xml +sed -i.bak 's/generated_phpdoc_test.php//g' tests/test.sh +sed -i.bak '/memory_leak_test.php/d' tests/test.sh for t in "${tests[@]}" do remove_error_test tests/$t diff --git a/php/tests/test.sh b/php/tests/test.sh index 6e70eb2a..700dd295 100755 --- a/php/tests/test.sh +++ b/php/tests/test.sh @@ -14,7 +14,7 @@ set -e phpize && ./configure CFLAGS='-g -O0' && make popd -tests=( array_test.php encode_decode_test.php generated_class_test.php generated_phpdoc_test.php map_field_test.php well_known_test.php generated_service_test.php descriptors_test.php ) +tests=( array_test.php encode_decode_test.php generated_class_test.php map_field_test.php well_known_test.php descriptors_test.php ) for t in "${tests[@]}" do diff --git a/protoc-artifacts/Dockerfile b/protoc-artifacts/Dockerfile index c346586b..95086eae 100644 --- a/protoc-artifacts/Dockerfile +++ b/protoc-artifacts/Dockerfile @@ -1,4 +1,4 @@ -FROM centos:6.6 +FROM centos:6.9 RUN yum install -y git \ tar \ @@ -28,15 +28,14 @@ RUN wget -q http://apache.cs.utah.edu/maven/maven-3/3.3.9/binaries/apache-maven- tar xz -C /var/local ENV PATH /var/local/apache-maven-3.3.9/bin:$PATH -# Install GCC 4.7 to support -static-libstdc++ -RUN wget http://people.centos.org/tru/devtools-1.1/devtools-1.1.repo -P /etc/yum.repos.d && \ - bash -c 'echo "enabled=1" >> /etc/yum.repos.d/devtools-1.1.repo' && \ - bash -c "sed -e 's/\$basearch/i386/g' /etc/yum.repos.d/devtools-1.1.repo > /etc/yum.repos.d/devtools-i386-1.1.repo" && \ - sed -e 's/testing-/testing-i386-/g' -i /etc/yum.repos.d/devtools-i386-1.1.repo && \ +# Install GCC 4.8 to support -static-libstdc++ +RUN wget http://people.centos.org/tru/devtools-2/devtools-2.repo -P /etc/yum.repos.d && \ + bash -c 'echo "enabled=1" >> /etc/yum.repos.d/devtools-2.repo' && \ + bash -c "sed -e 's/\$basearch/i386/g' /etc/yum.repos.d/devtools-2.repo > /etc/yum.repos.d/devtools-i386-2.repo" && \ + sed -e 's/testing-/testing-i386-/g' -i /etc/yum.repos.d/devtools-i386-2.repo && \ rpm --rebuilddb && \ - yum install -y devtoolset-1.1 \ - devtoolset-1.1-libstdc++-devel \ - devtoolset-1.1-libstdc++-devel.i686 && \ + yum install -y devtoolset-2-gcc devtoolset-2-gcc-c++ devtoolset-2-binutils devtoolset-2-libstdc++-devel \ + devtoolset-2-gcc.i686 devtoolset-2-gcc-c++.i686 devtoolset-2-binutils.i686 devtoolset-2-libstdc++-devel.i686 && \ yum clean all COPY scl-enable-devtoolset.sh /var/local/ diff --git a/protoc-artifacts/README.md b/protoc-artifacts/README.md index 5c5ca4ce..9b148891 100644 --- a/protoc-artifacts/README.md +++ b/protoc-artifacts/README.md @@ -7,13 +7,31 @@ build and publish a ``protoc`` executable (a.k.a. artifact) to Maven repositories. The artifact can be used by build automation tools so that users would not need to compile and install ``protoc`` for their systems. +If you would like us to publish protoc artifact for a new platform, please send +us a pull request to add support for the new platform. You would need to change +the following files: + +* [build-protoc.sh](build-protoc.sh): script to cross-build the protoc for your + platform. +* [pom.xml](pom.xml): script to upload artifacts to maven. +* [build-zip.sh](build-zip.sh): script to package published maven artifacts in + our release page. + +## Maven Location +The published protoc artifacts are available on Maven here: + + http://central.maven.org/maven2/com/google/protobuf/protoc/ + ## Versioning The version of the ``protoc`` artifact must be the same as the version of the Protobuf project. ## Artifact name The name of a published ``protoc`` artifact is in the following format: -``protoc-<version>-<os>-<arch>.exe``, e.g., ``protoc-3.0.0-alpha-3-windows-x86_64.exe``. +``protoc-<version>-<os>-<arch>.exe``, e.g., ``protoc-3.6.1-linux-x86_64.exe``. + +Note that artifacts for linux/macos also have the `.exe` suffix but they are +not windows binaries. ## System requirement Install [Apache Maven](http://maven.apache.org/) if you don't have it. @@ -29,98 +47,116 @@ generate the configure script. Under the protobuf project directory: + +``` +$ ./autogen.sh +``` + +### Build the artifact for each platform + +Run the build-protoc.sh script under this protoc-artifacts directory to build the protoc +artifact for each platform. For example: + ``` -$ ./autogen.sh && ./configure && make +$ cd protoc-artifacts +$ ./build-protoc.sh linux x86_64 protoc ``` -## To install artifacts locally -The following command will install the ``protoc`` artifact to your local Maven repository. +The above command will produce a `target/linux/x86_64/protoc` binary under the +protoc-artifacts directory. + +For a list of supported platforms, see the comments in the build-protoc.sh +script. We only use this script to build artifacts on Ubuntu and MacOS (both +with x86_64, and do cross-compilation for other platforms. + +### Tips for building for Linux +We build on Centos 6.9 to provide a good compatibility for not very new +systems. We have provided a ``Dockerfile`` under this directory to build the +environment. It has been tested with Docker 1.6.1. + +To build a image: + ``` -$ mvn install +$ docker build -t protoc-artifacts . ``` -## Cross-compilation -The Maven script will try to detect the OS and the architecture from Java -system properties. It's possible to build a protoc binary for an architecture -that is different from what Java has detected, as long as you have the proper -compilers installed. +To run the image: + +``` +$ docker run -it --rm=true protoc-artifacts bash +``` -You can override the Maven properties ``os.detected.name`` and -``os.detected.arch`` to force the script to generate binaries for a specific OS -and/or architecture. Valid values are defined as the return values of -``normalizeOs()`` and ``normalizeArch()`` of ``Detector`` from -[os-maven-plugin](https://github.com/trustin/os-maven-plugin/blob/master/src/main/java/kr/motd/maven/os/Detector.java). -Frequently used values are: -- ``os.detected.name``: ``linux``, ``osx``, ``windows``. -- ``os.detected.arch``: ``x86_32``, ``x86_64``, ``ppcle_64`` +To checkout protobuf (run within the container): -For example, MinGW32 only ships with 32-bit compilers, but you can still build -32-bit protoc under 64-bit Windows, with the following command: ``` -$ mvn install -Dos.detected.arch=x86_32 +$ # Replace v3.5.1 with the version you want +$ wget -O - https://github.com/google/protobuf/archive/v3.5.1.tar.gz | tar xvzp ``` +### Windows build +We no longer use scripts in this directory to build windows artifacts. Instead, +we use Visual Studio 2015 to build our windows release artifacts. See our +[kokoro windows build scripts here](../kokoro/release/protoc/windows/build.bat). + +To upload windows artifacts, copy the built binaries into this directory and +put it into the target/windows/(x86_64|x86_32) directory the same way as the +artifacts for other platforms. That will allow the maven script to find and +upload the artifacts to maven. + ## To push artifacts to Maven Central Before you can upload artifacts to Maven Central repository, make sure you have read [this page](http://central.sonatype.org/pages/apache-maven.html) on how to configure GPG and Sonatype account. -You need to perform the deployment for every platform that you want to -support. DO NOT close the staging repository until you have done the -deployment for all platforms. Currently the following platforms are supported: -- Linux (x86_32, x86_64, ppcle_64 and cross compiled aarch_64 or ppcle_64) -- Windows (x86_32 and x86_64) with - - Cygwin64 with MinGW compilers (x86_64) - - MSYS with MinGW32 (x86_32) - - Cross compile in Linux with MinGW-w64 (x86_32, x86_64) -- MacOSX (x86_32 and x86_64) - -As for MSYS2/MinGW64 for Windows: protoc will build, but it insists on -adding a dependency of `libwinpthread-1.dll`, which isn't shipped with -Windows. +Before you do the deployment, make sure you have built the protoc artifacts for +every supported platform and put them under the target directory. Example +target directory layout: + + + pom.xml + + target + + linux + + x86_64 + protoc.exe + + x86_32 + protoc.exe + + macos + + x86_64 + protoc.exe + + x86_32 + protoc.exe + + windows + + x86_64 + protoc.exe + + x86_32 + protoc.exe + +You will need to build the artifacts on multiple machines and gather them +together into one place. Use the following command to deploy artifacts for the host platform to a staging repository. + ``` $ mvn clean deploy -P release ``` + It creates a new staging repository. Go to https://oss.sonatype.org/#stagingRepositories and find the repository, usually -in the name like ``comgoogle-123``. - -You will want to run this command on a different platform. Remember, in -subsequent deployments you will need to provide the repository name that you -have found in the first deployment so that all artifacts go to the same -repository: -``` -$ mvn clean deploy -P release -Dstaging.repository=comgoogle-123 -``` - -A 32-bit artifact can be deployed from a 64-bit host with -``-Dos.detected.arch=x86_32`` - -An arm64 artifact can be deployed from x86 host with -``-Dos.detected.arch=aarch_64`` - -A ppcle_64 artifact can be deployed from x86 host with -``-Dos.detected.arch=ppcle_64`` - -A windows artifact can be deployed from a linux machine with -``-Dos.detected.name=windows`` - -When you have done deployment for all platforms, go to -https://oss.sonatype.org/#stagingRepositories, verify that the staging -repository has all the binaries, close and release this repository. +in the name like ``comgoogle-123``. Verify that the staging repository has all +the binaries, close and release this repository. ## Upload zip packages to github release page. After uploading protoc artifacts to Maven Central repository, run the build-zip.sh script to bulid zip packages for these protoc binaries and upload these zip packages to the download section of the github release. For example: + ``` $ ./build-zip.sh 3.6.0 ``` -The above command will create 6 zip files: + +The above command will create 5 zip files: + ``` dist/protoc-3.6.0-win32.zip dist/protoc-3.6.0-osx-x86_32.zip @@ -129,70 +165,14 @@ dist/protoc-3.6.0-linux-x86_32.zip dist/protoc-3.6.0-linux-x86_64.zip dist/protoc-3.6.0-linux-ppcle_64.zip ``` + Before running the script, make sure the artifacts are accessible from: http://repo1.maven.org/maven2/com/google/protobuf/protoc/ -### Tips for deploying on Linux -We build on Centos 6.6 to provide a good compatibility for not very new -systems. We have provided a ``Dockerfile`` under this directory to build the -environment. It has been tested with Docker 1.6.1. - -To build a image: -``` -$ docker build -t protoc-artifacts . -``` - -To run the image: -``` -$ docker run -it --rm=true protoc-artifacts bash -``` - -To checkout protobuf (run within the container): -``` -$ # Replace v3.5.1 with the version you want -$ wget -O - https://github.com/google/protobuf/archive/v3.5.1.tar.gz | tar xvzp -``` - -### Tips for deploying on Windows -Under Windows the following error may occur: ``gpg: cannot open tty `no tty': -No such file or directory``. This can be fixed by configuring gpg through an -active profile in ``.m2\settings.xml`` where also the Sonatype password is -stored: -```xml -<settings> - <servers> - <server> - <id>sonatype-nexus-staging</id> - <username>[username]</username> - <password>[password]</password> - </server> - </servers> - <profiles> - <profile> - <id>gpg</id> - <properties> - <gpg.executable>gpg</gpg.executable> - <gpg.passphrase>[password]</gpg.passphrase> - </properties> - </profile> - </profiles> - <activeProfiles> - <activeProfile>gpg</activeProfile> - </activeProfiles> -</settings> -``` - -### Tested build environments +## Tested build environments We have successfully built artifacts on the following environments: - Linux x86_32 and x86_64: - - Centos 6.6 (within Docker 1.6.1) - - Ubuntu 14.04.2 64-bit -- Linux ppc64le: - - Debian 9.4 - - Cross compiled with `g++-powerpc64le-linux-gnu` on Debian 9.4 x86_64 -- Linux aarch_64: Cross compiled with `g++-aarch64-linux-gnu` on Ubuntu 14.04.2 64-bit -- Windows x86_32: MSYS with ``mingw32-gcc-g++ 4.8.1-4`` on Windows 7 64-bit -- Windows x86_32: Cross compile with ``i686-w64-mingw32-g++ 4.8.2`` on Ubuntu 14.04.2 64-bit -- Windows x86_64: Cygwin64 with ``mingw64-x86_64-gcc-g++ 4.8.3-1`` on Windows 7 64-bit -- Windows x86_64: Cross compile with ``x86_64-w64-mingw32-g++ 4.8.2`` on Ubuntu 14.04.2 64-bit + - Centos 6.9 (within Docker 1.6.1) + - Ubuntu 14.04.5 64-bit +- Linux aarch_64: Cross compiled with `g++-aarch64-linux-gnu` on Ubuntu 14.04.5 64-bit - Mac OS X x86_32 and x86_64: Mac OS X 10.9.5 diff --git a/protoc-artifacts/build-protoc.sh b/protoc-artifacts/build-protoc.sh index c50a1207..b8d2eb87 100755 --- a/protoc-artifacts/build-protoc.sh +++ b/protoc-artifacts/build-protoc.sh @@ -1,24 +1,34 @@ #!/bin/bash -# Builds protoc executable into target/protoc.exe; optionally build protoc -# plugins into target/protoc-gen-*.exe -# To be run from Maven. -# Usage: build-protoc.sh <OS> <ARCH> <TARGET> -# <OS> and <ARCH> are ${os.detected.name} and ${os.detected.arch} from os-maven-plugin -# <TARGET> can be "protoc" or "protoc-gen-javalite" +# Builds protoc executable into target/<OS>/<ARCH>/protoc.exe; optionally builds +# protoc plugins into target/<OS>/<ARCH>/protoc-gen-*.exe # -# The script now supports cross-compiling windows, linux-arm64, and linux-ppc64le in linux-x86 -# environment. Required packages: -# - Windows: i686-w64-mingw32-gcc (32bit) and x86_64-w64-mingw32-gcc (64bit) -# - Arm64: g++-aarch64-linux-gnu -# - Ppc64le: g++-powerpc64le-linux-gcc +# Usage: ./build-protoc.sh <OS> <ARCH> <TARGET> +# +# <TARGET> can be "protoc" or "protoc-gen-javalite". Supported <OS> <ARCH> +# combinations: +# HOST <OS> <ARCH> <COMMENT> +# cygwin windows x86_32 Requires: i686-w64-mingw32-gcc +# cygwin windows x86_64 Requires: x86_64-w64-mingw32-gcc +# linux linux aarch_64 Requires: g++-aarch64-linux-gnu +# linux linux x86_32 +# linux linux x86_64 +# linux windows x86_32 Requires: i686-w64-mingw32-gcc +# linux windows x86_64 Requires: x86_64-w64-mingw32-gcc +# macos osx x86_32 +# macos osx x86_64 +# mingw windows x86_32 +# mingw windows x86_64 +# +# Before running this script, make sure you have generated the configure script +# in the parent directory (i.e., run ./autogen.sh there). OS=$1 ARCH=$2 MAKE_TARGET=$3 if [[ $# < 3 ]]; then - echo "No arguments provided. This script is intended to be run from Maven." + echo "Not enough arguments provided." exit 1 fi @@ -28,7 +38,7 @@ case $MAKE_TARGET in protoc) ;; *) - echo "Target ""$TARGET"" invalid." + echo "Target ""$MAKE_TARGET"" invalid." exit 1 esac @@ -140,7 +150,7 @@ checkDependencies () white_list="linux-vdso64\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|libz\.so\.1\|ld64\.so\.2" elif [[ "$ARCH" == aarch_64 ]]; then dump_cmd='objdump -p '"$1"' | grep NEEDED' - white_list="libpthread\.so\.0\|libc\.so\.6\|ld-linux-aarch64\.so\.1" + white_list="libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|ld-linux-aarch64\.so\.1" fi elif [[ "$OS" == osx ]]; then dump_cmd='otool -L '"$1"' | fgrep dylib' @@ -164,15 +174,10 @@ checkDependencies () } ############################################################################ -echo "Building protoc, OS=$OS ARCH=$ARCH TARGET=$TARGET" - -# Nested double quotes are unintuitive, but it works. -cd "$(dirname "$0")" +echo "Building protoc, OS=$OS ARCH=$ARCH TARGET=$MAKE_TARGET" -WORKING_DIR=$(pwd) CONFIGURE_ARGS="--disable-shared" -TARGET_FILE=target/$MAKE_TARGET.exe if [[ "$OS" == windows ]]; then MAKE_TARGET="${MAKE_TARGET}.exe" fi @@ -253,10 +258,18 @@ fi export CXXFLAGS LDFLAGS -cd "$WORKING_DIR"/.. && ./configure $CONFIGURE_ARGS && - cd src && make clean && make $MAKE_TARGET && - cd "$WORKING_DIR" && mkdir -p target && - cp ../src/$MAKE_TARGET $TARGET_FILE || +# Nested double quotes are unintuitive, but it works. +cd "$(dirname "$0")" + +WORKING_DIR="$(pwd)" +BUILD_DIR="build/$OS/$ARCH" +TARGET_FILE="target/$OS/$ARCH/$MAKE_TARGET.exe" + +mkdir -p "$BUILD_DIR" && cd "$BUILD_DIR" && + ../../../../configure $CONFIGURE_ARGS && + cd src && make $MAKE_TARGET -j8 && + cd "$WORKING_DIR" && mkdir -p $(dirname $TARGET_FILE) && + cp $BUILD_DIR/src/$MAKE_TARGET $TARGET_FILE || exit 1 if [[ "$OS" == osx ]]; then diff --git a/protoc-artifacts/pom.xml b/protoc-artifacts/pom.xml index 667a0215..390cd220 100644 --- a/protoc-artifacts/pom.xml +++ b/protoc-artifacts/pom.xml @@ -33,38 +33,9 @@ </connection> </scm> <build> - <extensions> - <extension> - <groupId>kr.motd.maven</groupId> - <artifactId>os-maven-plugin</artifactId> - <version>1.5.0.Final</version> - </extension> - </extensions> <plugins> <plugin> <groupId>org.codehaus.mojo</groupId> - <artifactId>exec-maven-plugin</artifactId> - <version>1.1.1</version> - <executions> - <execution> - <phase>compile</phase> - <goals> - <goal>exec</goal> - </goals> - </execution> - </executions> - <configuration> - <executable>bash</executable> - <arguments> - <argument>build-protoc.sh</argument> - <argument>${os.detected.name}</argument> - <argument>${os.detected.arch}</argument> - <argument>protoc</argument> - </arguments> - </configuration> - </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> <artifactId>build-helper-maven-plugin</artifactId> <version>1.8</version> <executions> @@ -77,8 +48,38 @@ <configuration> <artifacts> <artifact> - <file>${basedir}/target/protoc.exe</file> - <classifier>${os.detected.name}-${os.detected.arch}</classifier> + <file>${basedir}/target/linux/x86_64/protoc.exe</file> + <classifier>linux-x86_64</classifier> + <type>exe</type> + </artifact> + <artifact> + <file>${basedir}/target/linux/x86_32/protoc.exe</file> + <classifier>linux-x86_32</classifier> + <type>exe</type> + </artifact> + <artifact> + <file>${basedir}/target/windows/x86_64/protoc.exe</file> + <classifier>windows-x86_64</classifier> + <type>exe</type> + </artifact> + <artifact> + <file>${basedir}/target/windows/x86_32/protoc.exe</file> + <classifier>windows-x86_32</classifier> + <type>exe</type> + </artifact> + <artifact> + <file>${basedir}/target/osx/x86_64/protoc.exe</file> + <classifier>osx-x86_64</classifier> + <type>exe</type> + </artifact> + <artifact> + <file>${basedir}/target/osx/x86_32/protoc.exe</file> + <classifier>osx-x86_32</classifier> + <type>exe</type> + </artifact> + <artifact> + <file>${basedir}/target/linux/aarch_64/protoc.exe</file> + <classifier>linux-aarch_64</classifier> <type>exe</type> </artifact> </artifacts> diff --git a/protoc-artifacts/scl-enable-devtoolset.sh b/protoc-artifacts/scl-enable-devtoolset.sh index 8d9585ea..30895585 100755 --- a/protoc-artifacts/scl-enable-devtoolset.sh +++ b/protoc-artifacts/scl-enable-devtoolset.sh @@ -10,4 +10,4 @@ quote() { done } -exec scl enable devtoolset-1.1 "$(quote "$@")" +exec scl enable devtoolset-2 "$(quote "$@")" diff --git a/python/tox.ini b/python/tox.ini index 38a81b4f..4eb319d2 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -4,7 +4,8 @@ envlist = [testenv] usedevelop=true -passenv = CC +passenv = + CC KOKORO_BUILD_ID KOKORO_BUILD_NUMBER setenv = cpp: LD_LIBRARY_PATH={toxinidir}/../src/.libs cpp: DYLD_LIBRARY_PATH={toxinidir}/../src/.libs diff --git a/ruby/compatibility_tests/v3.0.0/tests/basic.rb b/ruby/compatibility_tests/v3.0.0/tests/basic.rb index 05fe0278..e2559036 100644 --- a/ruby/compatibility_tests/v3.0.0/tests/basic.rb +++ b/ruby/compatibility_tests/v3.0.0/tests/basic.rb @@ -222,33 +222,55 @@ module BasicTest def test_type_errors m = TestMessage.new - assert_raise TypeError do + + # Use rescue to allow subclasses of error + success = false + begin m.optional_int32 = "hello" + rescue TypeError + success = true end - assert_raise TypeError do - m.optional_string = 42 - end - assert_raise TypeError do + assert(success) + + success = false + begin m.optional_string = nil + rescue TypeError + success = true end - assert_raise TypeError do + assert(success) + + success = false + begin m.optional_bool = 42 + rescue TypeError + success = true end - assert_raise TypeError do + assert(success) + + success = false + begin m.optional_msg = TestMessage.new # expects TestMessage2 + rescue TypeError + success = true end + assert(success) - assert_raise TypeError do + success = false + begin m.repeated_int32 = [] # needs RepeatedField + rescue TypeError + success = true end + assert(success) - assert_raise TypeError do - m.repeated_int32.push "hello" - end - - assert_raise TypeError do + success = false + begin m.repeated_msg.push TestMessage.new + rescue TypeError + success = true end + assert(success) end def test_string_encoding @@ -275,7 +297,7 @@ module BasicTest # strings are immutable so we can't do this, but serialize should catch it. m.optional_string = "asdf".encode!('UTF-8') - assert_raise RuntimeError do + assert_raise do m.optional_string.encode!('ASCII-8BIT') end end @@ -312,10 +334,14 @@ module BasicTest assert l.pop == 9 assert l == [5, 2, 3, 4, 7, 8] - assert_raise TypeError do + success = false + begin m = TestMessage.new l.push m + rescue TypeError + success = true end + assert(success) m = TestMessage.new m.repeated_int32 = l @@ -362,12 +388,22 @@ module BasicTest l = Google::Protobuf::RepeatedField.new(:message, TestMessage) l.push TestMessage.new assert l.count == 1 - assert_raise TypeError do + + success = false + begin l.push TestMessage2.new + rescue TypeError + success = true end - assert_raise TypeError do + assert(success) + + success = false + begin l.push 42 + rescue TypeError + success = true end + assert(success) l2 = l.dup assert l2[0] == l[0] @@ -493,9 +529,14 @@ module BasicTest assert m.length == 0 assert m == {} - assert_raise TypeError do + success = false + begin m[1] = 1 + rescue TypeError + success = true end + assert(success) + assert_raise RangeError do m["asdf"] = 0x1_0000_0000 end @@ -514,18 +555,28 @@ module BasicTest assert_raise RangeError do m[0x8000_0000] = 1 end - assert_raise TypeError do + + success = false + begin m["asdf"] = 1 + rescue TypeError + success = true end + assert(success) m = Google::Protobuf::Map.new(:int64, :int32) m[0x1000_0000_0000_0000] = 1 assert_raise RangeError do m[0x1_0000_0000_0000_0000] = 1 end - assert_raise TypeError do + + success = false + begin m["asdf"] = 1 + rescue TypeError + success = true end + assert(success) m = Google::Protobuf::Map.new(:uint32, :int32) m[0x8000_0000] = 1 @@ -548,18 +599,32 @@ module BasicTest m = Google::Protobuf::Map.new(:bool, :int32) m[true] = 1 m[false] = 2 - assert_raise TypeError do + + success = false + begin m[1] = 1 + rescue TypeError + success = true end - assert_raise TypeError do + assert(success) + + success = false + begin m["asdf"] = 1 + rescue TypeError + success = true end + assert(success) m = Google::Protobuf::Map.new(:string, :int32) m["asdf"] = 1 - assert_raise TypeError do + success = false + begin m[1] = 1 + rescue TypeError + success = true end + assert(success) assert_raise Encoding::UndefinedConversionError do bytestring = ["FFFF"].pack("H*") m[bytestring] = 1 @@ -570,17 +635,25 @@ module BasicTest m[bytestring] = 1 # Allowed -- we will automatically convert to ASCII-8BIT. m["asdf"] = 1 - assert_raise TypeError do + success = false + begin m[1] = 1 + rescue TypeError + success = true end + assert(success) end def test_map_msg_enum_valuetypes m = Google::Protobuf::Map.new(:string, :message, TestMessage) m["asdf"] = TestMessage.new - assert_raise TypeError do + success = false + begin m["jkl;"] = TestMessage2.new + rescue TypeError + success = true end + assert(success) m = Google::Protobuf::Map.new( :string, :message, TestMessage, @@ -645,23 +718,39 @@ module BasicTest m.map_string_msg.delete("c") assert m.map_string_msg == { "a" => TestMessage2.new(:foo => 1) } - assert_raise TypeError do + success = false + begin m.map_string_msg["e"] = TestMessage.new # wrong value type + rescue TypeError + success = true end + assert(success) # ensure nothing was added by the above assert m.map_string_msg == { "a" => TestMessage2.new(:foo => 1) } m.map_string_int32 = Google::Protobuf::Map.new(:string, :int32) - assert_raise TypeError do + success = false + begin m.map_string_int32 = Google::Protobuf::Map.new(:string, :int64) + rescue TypeError + success = true end - assert_raise TypeError do + assert(success) + success = false + begin m.map_string_int32 = {} + rescue TypeError + success = true end + assert(success) - assert_raise TypeError do + success = false + begin m = MapMessage.new(:map_string_int32 => { 1 => "I am not a number" }) + rescue TypeError + success = true end + assert(success) end def test_map_encode_decode @@ -922,22 +1011,30 @@ module BasicTest def test_def_errors s = Google::Protobuf::DescriptorPool.new - assert_raise TypeError do + success = false + begin s.build do # enum with no default (integer value 0) add_enum "MyEnum" do value :A, 1 end end + rescue TypeError + success = true end - assert_raise TypeError do + assert(success) + success = false + begin s.build do # message with required field (unsupported in proto3) add_message "MyMessage" do required :foo, :int32, 1 end end + rescue TypeError + success = true end + assert(success) end def test_corecursive diff --git a/ruby/compatibility_tests/v3.0.0/tests/repeated_field_test.rb b/ruby/compatibility_tests/v3.0.0/tests/repeated_field_test.rb index 25727b7b..201fe36b 100644 --- a/ruby/compatibility_tests/v3.0.0/tests/repeated_field_test.rb +++ b/ruby/compatibility_tests/v3.0.0/tests/repeated_field_test.rb @@ -18,7 +18,7 @@ class RepeatedFieldTest < Test::Unit::TestCase # jRuby additions to the Array class that we can ignore arr_methods -= [ :indices, :iter_for_each, :iter_for_each_index, :iter_for_each_with_index, :dimensions, :copy_data, :copy_data_simple, - :nitems, :iter_for_reverse_each, :indexes] + :nitems, :iter_for_reverse_each, :indexes, :append, :prepend] arr_methods.each do |method_name| assert m.repeated_string.respond_to?(method_name) == true, "does not respond to #{method_name}" end diff --git a/ruby/ext/google/protobuf_c/storage.c b/ruby/ext/google/protobuf_c/storage.c index 1437c0b5..5d842b74 100644 --- a/ruby/ext/google/protobuf_c/storage.c +++ b/ruby/ext/google/protobuf_c/storage.c @@ -177,7 +177,7 @@ void native_slot_set_value_and_case(upb_fieldtype_t type, VALUE type_class, } case UPB_TYPE_STRING: if (CLASS_OF(value) == rb_cSymbol) { - value = rb_funcall(value, rb_intern("to_s"), 0, NULL); + value = rb_funcall(value, rb_intern("to_s"), 0); } else if (CLASS_OF(value) != rb_cString) { rb_raise(rb_eTypeError, "Invalid argument for string field."); } @@ -207,7 +207,7 @@ void native_slot_set_value_and_case(upb_fieldtype_t type, VALUE type_class, case UPB_TYPE_ENUM: { int32_t int_val = 0; if (TYPE(value) == T_STRING) { - value = rb_funcall(value, rb_intern("to_sym"), 0, NULL); + value = rb_funcall(value, rb_intern("to_sym"), 0); } else if (!is_ruby_num(value) && TYPE(value) != T_SYMBOL) { rb_raise(rb_eTypeError, "Expected number or symbol type for enum field."); diff --git a/ruby/tests/basic.rb b/ruby/tests/basic.rb index ad34d53d..d151022e 100644 --- a/ruby/tests/basic.rb +++ b/ruby/tests/basic.rb @@ -336,7 +336,9 @@ module BasicTest # strings are immutable so we can't do this, but serialize should catch it. m.optional_string = "asdf".encode!('UTF-8') - assert_raise RuntimeError do + # Ruby 2.5 changed to raise FrozenError. However, assert_raise don't + # accept subclass. Don't specify type here. + assert_raise do m.optional_string.encode!('ASCII-8BIT') end end diff --git a/ruby/tests/repeated_field_test.rb b/ruby/tests/repeated_field_test.rb index 61ac4afd..a4f3aab8 100644 --- a/ruby/tests/repeated_field_test.rb +++ b/ruby/tests/repeated_field_test.rb @@ -18,7 +18,7 @@ class RepeatedFieldTest < Test::Unit::TestCase # jRuby additions to the Array class that we can ignore arr_methods -= [ :indices, :iter_for_each, :iter_for_each_index, :iter_for_each_with_index, :dimensions, :copy_data, :copy_data_simple, - :nitems, :iter_for_reverse_each, :indexes] + :nitems, :iter_for_reverse_each, :indexes, :append, :prepend] arr_methods.each do |method_name| assert m.repeated_string.respond_to?(method_name) == true, "does not respond to #{method_name}" end diff --git a/src/Makefile.am b/src/Makefile.am index fb247300..09ab76f7 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -56,9 +56,7 @@ clean-local: CLEANFILES = $(protoc_outputs) unittest_proto_middleman \ testzip.jar testzip.list testzip.proto testzip.zip \ - no_warning_test.cc \ - google/protobuf/compiler/js/well_known_types_embed.cc \ - js_embed$(EXEEXT) + no_warning_test.cc MAINTAINERCLEANFILES = \ Makefile.in @@ -471,22 +469,6 @@ bin_PROGRAMS = protoc protoc_LDADD = $(PTHREAD_LIBS) libprotobuf.la libprotoc.la protoc_SOURCES = google/protobuf/compiler/main.cc -# The special JS code for the well-known types is linked into the compiler via -# well_known_types_embed.cc, which is generated from .js source files. We have -# to build the js_embed binary using $(CXX_FOR_BUILD) so that it is executable -# on the build machine in a cross-compilation setup. -js_embed$(EXEEXT): $(srcdir)/google/protobuf/compiler/js/embed.cc - $(CXX_FOR_BUILD) -o $@ $< -js_well_known_types_sources = \ - google/protobuf/compiler/js/well_known_types/any.js \ - google/protobuf/compiler/js/well_known_types/struct.js \ - google/protobuf/compiler/js/well_known_types/timestamp.js -# We have to cd to $(srcdir) so that out-of-tree builds work properly. -google/protobuf/compiler/js/well_known_types_embed.cc: js_embed$(EXEEXT) $(js_well_known_types_sources) - mkdir -p `dirname $@` && \ - oldpwd=`pwd` && cd $(srcdir) && \ - $$oldpwd/js_embed$(EXEEXT) $(js_well_known_types_sources) > $$oldpwd/$@ - # Tests ============================================================== protoc_inputs = \ @@ -565,7 +547,6 @@ EXTRA_DIST = \ google/protobuf/package_info.h \ google/protobuf/io/package_info.h \ google/protobuf/util/package_info.h \ - google/protobuf/compiler/js/embed.cc \ google/protobuf/compiler/ruby/ruby_generated_code.proto \ google/protobuf/compiler/ruby/ruby_generated_code_pb.rb \ google/protobuf/compiler/package_info.h \ diff --git a/src/google/protobuf/compiler/js/embed.cc b/src/google/protobuf/compiler/js/embed.cc deleted file mode 100644 index f0f946e5..00000000 --- a/src/google/protobuf/compiler/js/embed.cc +++ /dev/null @@ -1,110 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#include <cassert> -#include <cstdlib> -#include <fstream> -#include <iostream> -#include <string> - -static bool AsciiIsPrint(unsigned char c) { - return c >= 32 && c < 127; -} - -static char ToDecimalDigit(int num) { - assert(num < 10); - return '0' + num; -} - -static std::string CEscape(const std::string& str) { - std::string dest; - - for (size_t i = 0; i < str.size(); ++i) { - unsigned char ch = str[i]; - switch (ch) { - case '\n': dest += "\\n"; break; - case '\r': dest += "\\r"; break; - case '\t': dest += "\\t"; break; - case '\"': dest += "\\\""; break; - case '\\': dest += "\\\\"; break; - default: - if (AsciiIsPrint(ch)) { - dest += ch; - } else { - dest += "\\"; - dest += ToDecimalDigit(ch / 64); - dest += ToDecimalDigit((ch % 64) / 8); - dest += ToDecimalDigit(ch % 8); - } - break; - } - } - - return dest; -} - -static void AddFile(const char* name, std::basic_ostream<char>* out) { - std::ifstream in(name); - - if (!in.is_open()) { - std::cerr << "Couldn't open input file: " << name << "\n"; - std::exit(EXIT_FAILURE); - } - - // Make canonical name only include the final element. - for (const char *p = name; *p; p++) { - if (*p == '/') { - name = p + 1; - } - } - - *out << "{\"" << CEscape(name) << "\",\n"; - - for (std::string line; std::getline(in, line); ) { - *out << " \"" << CEscape(line) << "\\n\"\n"; - } - - *out << "},\n"; -} - -int main(int argc, char *argv[]) { - std::cout << "#include " - "\"google/protobuf/compiler/js/well_known_types_embed.h\"\n"; - std::cout << "struct FileToc well_known_types_js[] = {\n"; - - for (int i = 1; i < argc; i++) { - AddFile(argv[i], &std::cout); - } - - std::cout << " {NULL, NULL} // Terminate the list.\n"; - std::cout << "};\n"; - - return EXIT_SUCCESS; -} diff --git a/src/google/protobuf/compiler/js/well_known_types_embed.cc b/src/google/protobuf/compiler/js/well_known_types_embed.cc new file mode 100644 index 00000000..e5ee5510 --- /dev/null +++ b/src/google/protobuf/compiler/js/well_known_types_embed.cc @@ -0,0 +1,225 @@ +#include <google/protobuf/compiler/js/well_known_types_embed.h> + +struct FileToc well_known_types_js[] = { + {"any.js", + "/* This code will be inserted into generated code for\n" + " * google/protobuf/any.proto. */\n" + "\n" + "/**\n" + " * Returns the type name contained in this instance, if any.\n" + " * @return {string|undefined}\n" + " */\n" + "proto.google.protobuf.Any.prototype.getTypeName = function() {\n" + " return this.getTypeUrl().split('/').pop();\n" + "};\n" + "\n" + "\n" + "/**\n" + " * Packs the given message instance into this Any.\n" + " * @param {!Uint8Array} serialized The serialized data to pack.\n" + " * @param {string} name The type name of this message object.\n" + " * @param {string=} opt_typeUrlPrefix the type URL prefix.\n" + " */\n" + "proto.google.protobuf.Any.prototype.pack = function(serialized, name,\n" + " opt_typeUrlPrefix) " + "{\n" + " if (!opt_typeUrlPrefix) {\n" + " opt_typeUrlPrefix = 'type.googleapis.com/';\n" + " }\n" + "\n" + " if (opt_typeUrlPrefix.substr(-1) != '/') {\n" + " this.setTypeUrl(opt_typeUrlPrefix + '/' + name);\n" + " } else {\n" + " this.setTypeUrl(opt_typeUrlPrefix + name);\n" + " }\n" + "\n" + " this.setValue(serialized);\n" + "};\n" + "\n" + "\n" + "/**\n" + " * @template T\n" + " * Unpacks this Any into the given message object.\n" + " * @param {function(Uint8Array):T} deserialize Function that will " + "deserialize\n" + " * the binary data properly.\n" + " * @param {string} name The expected type name of this message object.\n" + " * @return {?T} If the name matched the expected name, returns the " + "deserialized\n" + " * object, otherwise returns null.\n" + " */\n" + "proto.google.protobuf.Any.prototype.unpack = function(deserialize, name) " + "{\n" + " if (this.getTypeName() == name) {\n" + " return deserialize(this.getValue_asU8());\n" + " } else {\n" + " return null;\n" + " }\n" + "};\n"}, + {"timestamp.js", + "/* This code will be inserted into generated code for\n" + " * google/protobuf/timestamp.proto. */\n" + "\n" + "/**\n" + " * Returns a JavaScript 'Date' object corresponding to this Timestamp.\n" + " * @return {!Date}\n" + " */\n" + "proto.google.protobuf.Timestamp.prototype.toDate = function() {\n" + " var seconds = this.getSeconds();\n" + " var nanos = this.getNanos();\n" + "\n" + " return new Date((seconds * 1000) + (nanos / 1000000));\n" + "};\n" + "\n" + "\n" + "/**\n" + " * Sets the value of this Timestamp object to be the given Date.\n" + " * @param {!Date} value The value to set.\n" + " */\n" + "proto.google.protobuf.Timestamp.prototype.fromDate = function(value) {\n" + " this.setSeconds(Math.floor(value.getTime() / 1000));\n" + " this.setNanos(value.getMilliseconds() * 1000000);\n" + "};\n"}, + {"struct.js", + "/* This code will be inserted into generated code for\n" + " * google/protobuf/struct.proto. */\n" + "\n" + "/**\n" + " * Typedef representing plain JavaScript values that can go into a\n" + " * Struct.\n" + " * @typedef {null|number|string|boolean|Array|Object}\n" + " */\n" + "proto.google.protobuf.JavaScriptValue;\n" + "\n" + "\n" + "/**\n" + " * Converts this Value object to a plain JavaScript value.\n" + " * @return {?proto.google.protobuf.JavaScriptValue} a plain JavaScript\n" + " * value representing this Struct.\n" + " */\n" + "proto.google.protobuf.Value.prototype.toJavaScript = function() {\n" + " var kindCase = proto.google.protobuf.Value.KindCase;\n" + " switch (this.getKindCase()) {\n" + " case kindCase.NULL_VALUE:\n" + " return null;\n" + " case kindCase.NUMBER_VALUE:\n" + " return this.getNumberValue();\n" + " case kindCase.STRING_VALUE:\n" + " return this.getStringValue();\n" + " case kindCase.BOOL_VALUE:\n" + " return this.getBoolValue();\n" + " case kindCase.STRUCT_VALUE:\n" + " return this.getStructValue().toJavaScript();\n" + " case kindCase.LIST_VALUE:\n" + " return this.getListValue().toJavaScript();\n" + " default:\n" + " throw new Error('Unexpected struct type');\n" + " }\n" + "};\n" + "\n" + "\n" + "/**\n" + " * Converts this JavaScript value to a new Value proto.\n" + " * @param {!proto.google.protobuf.JavaScriptValue} value The value to\n" + " * convert.\n" + " * @return {!proto.google.protobuf.Value} The newly constructed value.\n" + " */\n" + "proto.google.protobuf.Value.fromJavaScript = function(value) {\n" + " var ret = new proto.google.protobuf.Value();\n" + " switch (goog.typeOf(value)) {\n" + " case 'string':\n" + " ret.setStringValue(/** @type {string} */ (value));\n" + " break;\n" + " case 'number':\n" + " ret.setNumberValue(/** @type {number} */ (value));\n" + " break;\n" + " case 'boolean':\n" + " ret.setBoolValue(/** @type {boolean} */ (value));\n" + " break;\n" + " case 'null':\n" + " ret.setNullValue(proto.google.protobuf.NullValue.NULL_VALUE);\n" + " break;\n" + " case 'array':\n" + " ret.setListValue(proto.google.protobuf.ListValue.fromJavaScript(\n" + " /** @type{!Array} */ (value)));\n" + " break;\n" + " case 'object':\n" + " ret.setStructValue(proto.google.protobuf.Struct.fromJavaScript(\n" + " /** @type{!Object} */ (value)));\n" + " break;\n" + " default:\n" + " throw new Error('Unexpected struct type.');\n" + " }\n" + "\n" + " return ret;\n" + "};\n" + "\n" + "\n" + "/**\n" + " * Converts this ListValue object to a plain JavaScript array.\n" + " * @return {!Array} a plain JavaScript array representing this List.\n" + " */\n" + "proto.google.protobuf.ListValue.prototype.toJavaScript = function() {\n" + " var ret = [];\n" + " var values = this.getValuesList();\n" + "\n" + " for (var i = 0; i < values.length; i++) {\n" + " ret[i] = values[i].toJavaScript();\n" + " }\n" + "\n" + " return ret;\n" + "};\n" + "\n" + "\n" + "/**\n" + " * Constructs a ListValue protobuf from this plain JavaScript array.\n" + " * @param {!Array} array a plain JavaScript array\n" + " * @return {proto.google.protobuf.ListValue} a new ListValue object\n" + " */\n" + "proto.google.protobuf.ListValue.fromJavaScript = function(array) {\n" + " var ret = new proto.google.protobuf.ListValue();\n" + "\n" + " for (var i = 0; i < array.length; i++) {\n" + " " + "ret.addValues(proto.google.protobuf.Value.fromJavaScript(array[i]));\n" + " }\n" + "\n" + " return ret;\n" + "};\n" + "\n" + "\n" + "/**\n" + " * Converts this Struct object to a plain JavaScript object.\n" + " * @return {!Object<string, !proto.google.protobuf.JavaScriptValue>} a " + "plain\n" + " * JavaScript object representing this Struct.\n" + " */\n" + "proto.google.protobuf.Struct.prototype.toJavaScript = function() {\n" + " var ret = {};\n" + "\n" + " this.getFieldsMap().forEach(function(value, key) {\n" + " ret[key] = value.toJavaScript();\n" + " });\n" + "\n" + " return ret;\n" + "};\n" + "\n" + "\n" + "/**\n" + " * Constructs a Struct protobuf from this plain JavaScript object.\n" + " * @param {!Object} obj a plain JavaScript object\n" + " * @return {proto.google.protobuf.Struct} a new Struct object\n" + " */\n" + "proto.google.protobuf.Struct.fromJavaScript = function(obj) {\n" + " var ret = new proto.google.protobuf.Struct();\n" + " var map = ret.getFieldsMap();\n" + "\n" + " for (var property in obj) {\n" + " var val = obj[property];\n" + " map.set(property, proto.google.protobuf.Value.fromJavaScript(val));\n" + " }\n" + "\n" + " return ret;\n" + "};\n"}, + {NULL, NULL} // Terminate the list. +}; @@ -1,14 +1,8 @@ #!/bin/bash # -# Build and runs tests for the protobuf project. The tests as written here are -# used by both Jenkins and Travis, though some specialized logic is required to -# handle the differences between them. - -on_travis() { - if [ "$TRAVIS" == "true" ]; then - "$@" - fi -} +# Build and runs tests for the protobuf project. We use this script to run +# tests on kokoro (Ubuntu and MacOS). It can run locally as well but you +# will need to make sure the required compilers/tools are available. # For when some other test needs the C++ main build, including protoc and # libprotobuf. @@ -18,15 +12,6 @@ internal_build_cpp() { return fi - if [[ $(uname -s) == "Linux" && "$TRAVIS" == "true" ]]; then - # Install GCC 4.8 to replace the default GCC 4.6. We need 4.8 for more - # decent C++ 11 support in order to compile conformance tests. - sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y - sudo apt-get update -qq - sudo apt-get install -qq g++-4.8 - export CXX="g++-4.8" CC="gcc-4.8" - fi - # Initialize any submodules. git submodule update --init --recursive @@ -43,8 +28,6 @@ build_cpp() { # The benchmark code depends on cmake, so test if it is installed before # trying to do the build. - # NOTE: The travis macOS images say they have cmake, but the xcode8.1 image - # appears to be missing it: https://github.com/travis-ci/travis-ci/issues/6996 if [[ $(type cmake 2>/dev/null) ]]; then # Verify benchmarking code can build successfully. cd benchmarks && make cpp-benchmark && cd .. @@ -121,10 +104,6 @@ build_golang() { # Add protoc to the path so that the examples build finds it. export PATH="`pwd`/src:$PATH" - # Install Go and the Go protobuf compiler plugin. - on_travis sudo apt-get update -qq - on_travis sudo apt-get install -qq golang - export GOPATH="$HOME/gocode" mkdir -p "$GOPATH/src/github.com/google" rm -f "$GOPATH/src/github.com/google/protobuf" @@ -139,27 +118,17 @@ use_java() { version=$1 case "$version" in jdk7) - on_travis sudo apt-get install openjdk-7-jdk export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 ;; oracle7) - if [ "$TRAVIS" == "true" ]; then - sudo apt-get install python-software-properties # for apt-add-repository - echo "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" | \ - sudo debconf-set-selections - yes | sudo apt-add-repository ppa:webupd8team/java - yes | sudo apt-get install oracle-java7-installer - fi; export PATH=/usr/lib/jvm/java-7-oracle/bin:$PATH export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 ;; esac - if [ "$TRAVIS" != "true" ]; then - MAVEN_LOCAL_REPOSITORY=/var/maven_local_repository - MVN="$MVN -e -X --offline -Dmaven.repo.local=$MAVEN_LOCAL_REPOSITORY" - fi; + MAVEN_LOCAL_REPOSITORY=/var/maven_local_repository + MVN="$MVN -e -X --offline -Dmaven.repo.local=$MAVEN_LOCAL_REPOSITORY" which java java -version @@ -190,12 +159,6 @@ build_java_with_conformance_tests() { cd conformance && make test_java && cd .. } -build_javanano() { - # Java build needs `protoc`. - internal_build_cpp - cd javanano && $MVN test && cd .. -} - build_java_jdk7() { use_java jdk7 build_java_with_conformance_tests @@ -213,38 +176,6 @@ build_java_compatibility() { ./test.sh 3.0.0-beta-4 } -build_javanano_jdk7() { - use_java jdk7 - build_javanano -} -build_javanano_oracle7() { - use_java oracle7 - build_javanano -} - -internal_install_python_deps() { - if [ "$TRAVIS" != "true" ]; then - return; - fi - # Install tox (OS X doesn't have pip). - if [ $(uname -s) == "Darwin" ]; then - brew upgrade python - python3 -m pip install tox - else - sudo pip install tox - fi - # Only install Python2.6/3.x on Linux. - if [ $(uname -s) == "Linux" ]; then - sudo apt-get install -y python-software-properties # for apt-add-repository - sudo apt-add-repository -y ppa:fkrull/deadsnakes - sudo apt-get update -qq - sudo apt-get install -y python3.3 python3.3-dev - sudo apt-get install -y python3.4 python3.4-dev - sudo apt-get install -y python3.5 python3.5-dev - sudo apt-get install -y python3.6 python3.6-dev - fi -} - build_objectivec_ios() { # Reused the build script that takes care of configuring and ensuring things # are up to date. The OS X test runs the objc conformance test, so skip it @@ -279,9 +210,7 @@ build_objectivec_cocoapods_integration() { build_python() { internal_build_cpp - internal_install_python_deps cd python - # Only test Python 2.6/3.x on Linux if [ $(uname -s) == "Linux" ]; then envlist=py\{27,33,34,35,36\}-python else @@ -293,11 +222,9 @@ build_python() { build_python_cpp() { internal_build_cpp - internal_install_python_deps export LD_LIBRARY_PATH=../src/.libs # for Linux export DYLD_LIBRARY_PATH=../src/.libs # for OS X cd python - # Only test Python 3.x on Linux if [ $(uname -s) == "Linux" ]; then envlist=py\{27,33,34,35,36\}-cpp else @@ -325,18 +252,24 @@ build_ruby22() { internal_build_cpp # For conformance tests. cd ruby && bash travis-test.sh ruby-2.2 && cd .. } -build_jruby() { +build_ruby23() { + internal_build_cpp # For conformance tests. + cd ruby && bash travis-test.sh ruby-2.3 && cd .. +} +build_ruby24() { internal_build_cpp # For conformance tests. - # TODO(xiaofeng): Upgrade to jruby-9.x. There are some broken jests to be - # fixed. - cd ruby && bash travis-test.sh jruby-1.7 && cd .. + cd ruby && bash travis-test.sh ruby-2.4 && cd .. +} +build_ruby25() { + internal_build_cpp # For conformance tests. + cd ruby && bash travis-test.sh ruby-2.5.0 && cd .. } build_ruby_all() { build_ruby21 build_ruby22 - # TODO(teboring): Disable jruby test temperarily for it randomly fails. - # https://grpc-testing.appspot.com/job/protobuf_pull_request/735/consoleFull. - # build_jruby + build_ruby23 + build_ruby24 + build_ruby25 } build_javascript() { @@ -617,14 +550,6 @@ build_php_all() { build_php_compatibility } -# Note: travis currently does not support testing more than one language so the -# .travis.yml cheats and claims to only be cpp. If they add multiple language -# support, this should probably get updated to install steps and/or -# rvm/gemfile/jdk/etc. entries rather than manually doing the work. - -# .travis.yml uses matrix.exclude to block the cases where app-get can't be -# use to install things. - # -------- main -------- if [ "$#" -ne 1 ]; then @@ -635,8 +560,6 @@ Usage: $0 { cpp | java_jdk7 | java_oracle7 | java_compatibility | - javanano_jdk7 | - javanano_oracle7 | objectivec_ios | objectivec_ios_debug | objectivec_ios_release | @@ -665,4 +588,5 @@ fi set -e # exit immediately on error set -x # display all commands +cd $(dirname $0) eval "build_$1" |