diff options
57 files changed, 1235 insertions, 32 deletions
diff --git a/java/core/src/test/java/com/google/protobuf/DeprecatedFieldTest.java b/java/core/src/test/java/com/google/protobuf/DeprecatedFieldTest.java index 2c272a73..9c0997c4 100644 --- a/java/core/src/test/java/com/google/protobuf/DeprecatedFieldTest.java +++ b/java/core/src/test/java/com/google/protobuf/DeprecatedFieldTest.java @@ -37,22 +37,22 @@ import junit.framework.TestCase; /** * Test field deprecation - * + * * @author birdo@google.com (Roberto Scaramuzzi) */ public class DeprecatedFieldTest extends TestCase { private String[] deprecatedGetterNames = { "hasDeprecatedInt32", "getDeprecatedInt32"}; - + private String[] deprecatedBuilderGetterNames = { "hasDeprecatedInt32", "getDeprecatedInt32", "clearDeprecatedInt32"}; - + private String[] deprecatedBuilderSetterNames = { - "setDeprecatedInt32"}; - + "setDeprecatedInt32"}; + public void testDeprecatedField() throws Exception { Class<?> deprecatedFields = TestDeprecatedFields.class; Class<?> deprecatedFieldsBuilder = TestDeprecatedFields.Builder.class; @@ -72,7 +72,15 @@ public class DeprecatedFieldTest extends TestCase { isDeprecated(method)); } } - + + public void testDeprecatedFieldInOneof() throws Exception { + Class<?> oneofCase = TestDeprecatedFields.OneofFieldsCase.class; + String name = "DEPRECATED_INT32_IN_ONEOF"; + java.lang.reflect.Field enumValue = oneofCase.getField(name); + assertTrue("Enum value " + name + " should be deprecated.", + isDeprecated(enumValue)); + } + private boolean isDeprecated(AnnotatedElement annotated) { return annotated.isAnnotationPresent(Deprecated.class); } diff --git a/kokoro/linux/32-bit/Dockerfile b/kokoro/linux/32-bit/Dockerfile new file mode 100644 index 00000000..1278889f --- /dev/null +++ b/kokoro/linux/32-bit/Dockerfile @@ -0,0 +1,143 @@ +# This Dockerfile specifies the recipe for creating an image for the tests +# to run in. +# +# We install as many test dependencies here as we can, because these setup +# steps can be cached. They do *not* run every time we run the build. +# The Docker image is only rebuilt when the Dockerfile (ie. this file) +# changes. + +# Base Dockerfile for gRPC dev images +FROM 32bit/debian:latest + +# Apt source for php +RUN echo "deb http://ppa.launchpad.net/ondrej/php/ubuntu trusty main" | tee /etc/apt/sources.list.d/various-php.list && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F4FCBB07 + +# Install dependencies. We start with the basic ones require to build protoc +# and the C++ build +RUN apt-get clean && apt-get update && apt-get install -y --force-yes \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + gcc \ + git \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + parallel \ + time \ + wget \ + unzip \ + # -- For python -- + python-setuptools \ + python-pip \ + python-dev \ + # -- For C++ benchmarks -- + cmake \ + # -- For PHP -- + php5.5 \ + php5.5-dev \ + php5.5-xml \ + php5.6 \ + php5.6-dev \ + php5.6-xml \ + php7.0 \ + php7.0-dev \ + php7.0-xml \ + phpunit \ + valgrind \ + libxml2-dev \ + && apt-get clean + +################## +# PHP dependencies. +RUN wget http://am1.php.net/get/php-5.5.38.tar.bz2/from/this/mirror +RUN mv mirror php-5.5.38.tar.bz2 +RUN tar -xvf php-5.5.38.tar.bz2 +RUN cd php-5.5.38 && ./configure --enable-maintainer-zts --prefix=/usr/local/php-5.5-zts && \ + make && make install && make clean && cd .. +RUN cd php-5.5.38 && make clean && ./configure --enable-bcmath --prefix=/usr/local/php-5.5 && \ + make && make install && make clean && cd .. + +RUN wget http://am1.php.net/get/php-5.6.30.tar.bz2/from/this/mirror +RUN mv mirror php-5.6.30.tar.bz2 +RUN tar -xvf php-5.6.30.tar.bz2 +RUN cd php-5.6.30 && ./configure --enable-maintainer-zts --prefix=/usr/local/php-5.6-zts && \ + make && make install && cd .. +RUN cd php-5.6.30 && make clean && ./configure --enable-bcmath --prefix=/usr/local/php-5.6 && \ + make && make install && cd .. + +RUN wget http://am1.php.net/get/php-7.0.18.tar.bz2/from/this/mirror +RUN mv mirror php-7.0.18.tar.bz2 +RUN tar -xvf php-7.0.18.tar.bz2 +RUN cd php-7.0.18 && ./configure --enable-maintainer-zts --prefix=/usr/local/php-7.0-zts && \ + make && make install && cd .. +RUN cd php-7.0.18 && make clean && ./configure --enable-bcmath --prefix=/usr/local/php-7.0 && \ + make && make install && cd .. + +RUN wget http://am1.php.net/get/php-7.1.4.tar.bz2/from/this/mirror +RUN mv mirror php-7.1.4.tar.bz2 +RUN tar -xvf php-7.1.4.tar.bz2 +RUN cd php-7.1.4 && ./configure --enable-maintainer-zts --prefix=/usr/local/php-7.1-zts && \ + make && make install && cd .. +RUN cd php-7.1.4 && make clean && ./configure --enable-bcmath --prefix=/usr/local/php-7.1 && \ + make && make install && cd .. + +RUN php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" +RUN php composer-setup.php +RUN mv composer.phar /usr/bin/composer +RUN php -r "unlink('composer-setup.php');" +RUN composer config -g -- disable-tls true +RUN composer config -g -- secure-http false +RUN cd /tmp && \ + git clone https://github.com/google/protobuf.git && \ + cd protobuf/php && \ + git reset --hard 49b44bff2b6257a119f9c6a342d6151c736586b8 && \ + ln -sfn /usr/local/php-5.5/bin/php /usr/bin/php && \ + ln -sfn /usr/local/php-5.5/bin/php-config /usr/bin/php-config && \ + ln -sfn /usr/local/php-5.5/bin/phpize /usr/bin/phpize && \ + composer install && \ + mv vendor /usr/local/vendor-5.5 && \ + ln -sfn /usr/local/php-5.6/bin/php /usr/bin/php && \ + ln -sfn /usr/local/php-5.6/bin/php-config /usr/bin/php-config && \ + ln -sfn /usr/local/php-5.6/bin/phpize /usr/bin/phpize && \ + composer install && \ + mv vendor /usr/local/vendor-5.6 && \ + ln -sfn /usr/local/php-7.0/bin/php /usr/bin/php && \ + ln -sfn /usr/local/php-7.0/bin/php-config /usr/bin/php-config && \ + ln -sfn /usr/local/php-7.0/bin/phpize /usr/bin/phpize && \ + composer install && \ + mv vendor /usr/local/vendor-7.0 && \ + ln -sfn /usr/local/php-7.1/bin/php /usr/bin/php && \ + ln -sfn /usr/local/php-7.1/bin/php-config /usr/bin/php-config && \ + ln -sfn /usr/local/php-7.1/bin/phpize /usr/bin/phpize && \ + composer install && \ + mv vendor /usr/local/vendor-7.1 + +################## +# Python dependencies + +# These packages exist in apt-get, but their versions are too old, so we have +# to get updates from pip. + +RUN pip install pip --upgrade +RUN pip install virtualenv tox yattag + +################## +# Prepare ccache + +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + +# Define the default command. +CMD ["bash"] diff --git a/kokoro/linux/32-bit/build.sh b/kokoro/linux/32-bit/build.sh new file mode 100755 index 00000000..04383a57 --- /dev/null +++ b/kokoro/linux/32-bit/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request 32" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERFILE_DIR=kokoro/linux/32-bit +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="php_all_32" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/32-bit/continuous.cfg b/kokoro/linux/32-bit/continuous.cfg new file mode 100644 index 00000000..28b66c07 --- /dev/null +++ b/kokoro/linux/32-bit/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/32-bit/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/32-bit/presubmit.cfg b/kokoro/linux/32-bit/presubmit.cfg new file mode 100644 index 00000000..28b66c07 --- /dev/null +++ b/kokoro/linux/32-bit/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/32-bit/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/64-bit/Dockerfile b/kokoro/linux/64-bit/Dockerfile new file mode 100644 index 00000000..3a279e66 --- /dev/null +++ b/kokoro/linux/64-bit/Dockerfile @@ -0,0 +1,244 @@ +# This Dockerfile specifies the recipe for creating an image for the tests +# to run in. +# +# We install as many test dependencies here as we can, because these setup +# steps can be cached. They do *not* run every time we run the build. +# The Docker image is only rebuilt when the Dockerfile (ie. this file) +# changes. + +# Base Dockerfile for gRPC dev images +FROM debian:latest + +# Apt source for old Python versions. +RUN echo 'deb http://ppa.launchpad.net/fkrull/deadsnakes/ubuntu trusty main' > /etc/apt/sources.list.d/deadsnakes.list && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys DB82666C + +# Apt source for Oracle Java. +RUN echo 'deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main' > /etc/apt/sources.list.d/webupd8team-java-trusty.list && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EEA14886 && \ + echo "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections + +# Apt source for Mono +RUN echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list && \ + echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF + +# Apt source for php +RUN echo "deb http://ppa.launchpad.net/ondrej/php/ubuntu trusty main" | tee /etc/apt/sources.list.d/various-php.list && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F4FCBB07 + +# Install dotnet SDK based on https://www.microsoft.com/net/core#debian +# (Ubuntu instructions need apt to support https) +RUN apt-get update && apt-get install -y --force-yes curl libunwind8 gettext && \ + curl -sSL -o dotnet.tar.gz https://go.microsoft.com/fwlink/?LinkID=847105 && \ + mkdir -p /opt/dotnet && tar zxf dotnet.tar.gz -C /opt/dotnet && \ + ln -s /opt/dotnet/dotnet /usr/local/bin + +# Install dependencies. We start with the basic ones require to build protoc +# and the C++ build +RUN apt-get clean && apt-get update && apt-get install -y --force-yes \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + gcc \ + git \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + parallel \ + time \ + wget \ + # -- For csharp -- + mono-devel \ + referenceassemblies-pcl \ + nunit \ + # -- For all Java builds -- \ + maven \ + # -- For java_jdk6 -- \ + # oops! not in jessie. too old? openjdk-6-jdk \ + # -- For java_jdk7 -- \ + openjdk-7-jdk \ + # -- For java_oracle7 -- \ + oracle-java7-installer \ + # -- For python / python_cpp -- \ + python-setuptools \ + python-pip \ + python-dev \ + python2.6-dev \ + python3.3-dev \ + python3.4-dev \ + # -- For Ruby -- + ruby \ + # -- For C++ benchmarks -- + cmake \ + # -- For PHP -- + php5.6 \ + php5.6-dev \ + php5.6-xml \ + php7.0 \ + php7.0-dev \ + php7.0-xml \ + phpunit \ + valgrind \ + libxml2-dev \ + && apt-get clean + +################## +# C# dependencies + +RUN wget www.nuget.org/NuGet.exe -O /usr/local/bin/nuget.exe + +################## +# Python dependencies + +# These packages exist in apt-get, but their versions are too old, so we have +# to get updates from pip. + +RUN pip install pip --upgrade +RUN pip install virtualenv tox yattag + +################## +# Ruby dependencies + +# Install rvm +RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 +RUN \curl -sSL https://get.rvm.io | bash -s stable + +# Install Ruby 2.1, Ruby 2.2 and JRuby 1.7 +RUN /bin/bash -l -c "rvm install ruby-2.1" +RUN /bin/bash -l -c "rvm install ruby-2.2" +RUN /bin/bash -l -c "rvm install jruby-1.7" +RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc" +RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" +RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" + +################## +# Java dependencies + +# This step requires compiling protoc. :( + +ENV MAVEN_REPO /var/maven_local_repository +ENV MVN mvn --batch-mode + +RUN cd /tmp && \ + git clone https://github.com/google/protobuf.git && \ + cd protobuf && \ + git reset --hard 129a6e2aca95dcfb6c3e717d7b9cca1f104fde39 && \ + ./autogen.sh && \ + ./configure && \ + make -j4 && \ + cd java && \ + $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO && \ + cd ../javanano && \ + $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO + +################## +# PHP dependencies. +RUN wget http://am1.php.net/get/php-5.5.38.tar.bz2/from/this/mirror +RUN mv mirror php-5.5.38.tar.bz2 +RUN tar -xvf php-5.5.38.tar.bz2 +RUN cd php-5.5.38 && ./configure --enable-maintainer-zts --prefix=/usr/local/php-5.5-zts && \ + make && make install && cd .. +RUN cd php-5.5.38 && make clean && ./configure --enable-bcmath --prefix=/usr/local/php-5.5 && \ + make && make install && cd .. + +RUN wget http://am1.php.net/get/php-5.6.30.tar.bz2/from/this/mirror +RUN mv mirror php-5.6.30.tar.bz2 +RUN tar -xvf php-5.6.30.tar.bz2 +RUN cd php-5.6.30 && ./configure --enable-maintainer-zts --prefix=/usr/local/php-5.6-zts && \ + make && make install && cd .. +RUN cd php-5.6.30 && make clean && ./configure --enable-bcmath --prefix=/usr/local/php-5.6 && \ + make && make install && cd .. + +RUN wget http://am1.php.net/get/php-7.0.18.tar.bz2/from/this/mirror +RUN mv mirror php-7.0.18.tar.bz2 +RUN tar -xvf php-7.0.18.tar.bz2 +RUN cd php-7.0.18 && ./configure --enable-maintainer-zts --prefix=/usr/local/php-7.0-zts && \ + make && make install && cd .. +RUN cd php-7.0.18 && make clean && ./configure --enable-bcmath --prefix=/usr/local/php-7.0 && \ + make && make install && cd .. + +RUN wget http://am1.php.net/get/php-7.1.4.tar.bz2/from/this/mirror +RUN mv mirror php-7.1.4.tar.bz2 +RUN tar -xvf php-7.1.4.tar.bz2 +RUN cd php-7.1.4 && ./configure --enable-maintainer-zts --prefix=/usr/local/php-7.1-zts && \ + make && make install && cd .. +RUN cd php-7.1.4 && make clean && ./configure --enable-bcmath --prefix=/usr/local/php-7.1 && \ + make && make install && cd .. + +RUN php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" +RUN php composer-setup.php +RUN mv composer.phar /usr/bin/composer +RUN php -r "unlink('composer-setup.php');" +RUN composer config -g -- disable-tls true +RUN composer config -g -- secure-http false +RUN cd /tmp && \ + rm -rf protobuf && \ + git clone https://github.com/google/protobuf.git && \ + cd protobuf && \ + git reset --hard 49b44bff2b6257a119f9c6a342d6151c736586b8 && \ + cd php && \ + ln -sfn /usr/local/php-5.5/bin/php /usr/bin/php && \ + ln -sfn /usr/local/php-5.5/bin/php-config /usr/bin/php-config && \ + ln -sfn /usr/local/php-5.5/bin/phpize /usr/bin/phpize && \ + composer install && \ + mv vendor /usr/local/vendor-5.5 && \ + ln -sfn /usr/local/php-5.6/bin/php /usr/bin/php && \ + ln -sfn /usr/local/php-5.6/bin/php-config /usr/bin/php-config && \ + ln -sfn /usr/local/php-5.6/bin/phpize /usr/bin/phpize && \ + composer install && \ + mv vendor /usr/local/vendor-5.6 && \ + ln -sfn /usr/local/php-7.0/bin/php /usr/bin/php && \ + ln -sfn /usr/local/php-7.0/bin/php-config /usr/bin/php-config && \ + ln -sfn /usr/local/php-7.0/bin/phpize /usr/bin/phpize && \ + composer install && \ + mv vendor /usr/local/vendor-7.0 && \ + ln -sfn /usr/local/php-7.1/bin/php /usr/bin/php && \ + ln -sfn /usr/local/php-7.1/bin/php-config /usr/bin/php-config && \ + ln -sfn /usr/local/php-7.1/bin/phpize /usr/bin/phpize && \ + composer install && \ + mv vendor /usr/local/vendor-7.1 + +################## +# Go dependencies. +RUN apt-get install -y \ + # -- For go -- \ + golang + +################## +# Javascript dependencies. +RUN apt-get install -y \ + # -- For javascript -- \ + npm + +################## +# Python 3.5 3.6 dependencies. +RUN apt-get clean && apt-get update && apt-get install -y --force-yes \ + python3.5-dev \ + python3.6-dev \ + && apt-get clean + +# On Debian/Ubuntu, nodejs binary is named 'nodejs' because the name 'node' +# is taken by another legacy binary. We don't have that legacy binary and +# npm expects the binary to be named 'node', so we just create a symbol +# link here. +RUN ln -s `which nodejs` /usr/bin/node + +################## +# Prepare ccache + +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + +# Define the default command. +CMD ["bash"] diff --git a/kokoro/linux/64-bit/build.sh b/kokoro/linux/64-bit/build.sh new file mode 100755 index 00000000..48ddbce5 --- /dev/null +++ b/kokoro/linux/64-bit/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERFILE_DIR=kokoro/linux/64-bit +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="csharp java_jdk7 javanano_jdk7 java_oracle7 javanano_oracle7 python python_cpp ruby_all javascript golang php_all" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/64-bit/continuous.cfg b/kokoro/linux/64-bit/continuous.cfg new file mode 100644 index 00000000..3a4faac1 --- /dev/null +++ b/kokoro/linux/64-bit/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/64-bit/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/64-bit/presubmit.cfg b/kokoro/linux/64-bit/presubmit.cfg new file mode 100644 index 00000000..3a4faac1 --- /dev/null +++ b/kokoro/linux/64-bit/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/64-bit/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/build_and_run_docker.sh b/kokoro/linux/build_and_run_docker.sh new file mode 100755 index 00000000..6a1f327a --- /dev/null +++ b/kokoro/linux/build_and_run_docker.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# +# Builds docker image and runs a command under it. +# This is a generic script that is configured with the following variables: +# +# DOCKERFILE_DIR - Directory in which Dockerfile file is located. +# DOCKER_RUN_SCRIPT - Script to run under docker (relative to protobuf repo root) +# OUTPUT_DIR - Directory that will be copied from inside docker after finishing. +# $@ - Extra args to pass to docker run + + +set -ex + +cd $(dirname $0)/../.. +git_root=$(pwd) +cd - + +# Use image name based on Dockerfile sha1 +DOCKERHUB_ORGANIZATION=grpctesting/protobuf +DOCKER_IMAGE_NAME=${DOCKERHUB_ORGANIZATION}_$(sha1sum $DOCKERFILE_DIR/Dockerfile | cut -f1 -d\ ) + +# Pull dockerimage from Dockerhub +docker pull $DOCKER_IMAGE_NAME + +# Ensure existence of ccache directory +CCACHE_DIR=/tmp/protobuf-ccache +mkdir -p $CCACHE_DIR + +# Choose random name for docker container +CONTAINER_NAME="build_and_run_docker_$(uuidgen)" + +echo $git_root + +# Run command inside docker +docker run \ + "$@" \ + -e CCACHE_DIR=$CCACHE_DIR \ + -e EXTERNAL_GIT_ROOT="/var/local/kokoro/protobuf" \ + -e TEST_SET="$TEST_SET" \ + -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \ + -v "$git_root:/var/local/kokoro/protobuf:ro" \ + -v $CCACHE_DIR:$CCACHE_DIR \ + -w /var/local/git/protobuf \ + --name=$CONTAINER_NAME \ + $DOCKER_IMAGE_NAME \ + bash -l "/var/local/kokoro/protobuf/$DOCKER_RUN_SCRIPT" || FAILED="true" + +# Copy output artifacts +if [ "$OUTPUT_DIR" != "" ] +then + docker cp "$CONTAINER_NAME:/var/local/git/protobuf/$OUTPUT_DIR" "${git_root}/kokoro" || FAILED="true" +fi + +# remove the container, possibly killing it first +docker rm -f $CONTAINER_NAME || true + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/kokoro/linux/golang/build.sh b/kokoro/linux/golang/build.sh new file mode 100755 index 00000000..79cddff1 --- /dev/null +++ b/kokoro/linux/golang/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERFILE_DIR=kokoro/linux/64-bit +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="golang" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/golang/continuous.cfg b/kokoro/linux/golang/continuous.cfg new file mode 100644 index 00000000..2282f78f --- /dev/null +++ b/kokoro/linux/golang/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/golang/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/golang/presubmit.cfg b/kokoro/linux/golang/presubmit.cfg new file mode 100644 index 00000000..2282f78f --- /dev/null +++ b/kokoro/linux/golang/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/golang/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/java_jdk7/build.sh b/kokoro/linux/java_jdk7/build.sh new file mode 100755 index 00000000..0a5604b4 --- /dev/null +++ b/kokoro/linux/java_jdk7/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERFILE_DIR=kokoro/linux/64-bit +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="java_jdk7" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/java_jdk7/continuous.cfg b/kokoro/linux/java_jdk7/continuous.cfg new file mode 100644 index 00000000..07a9df5f --- /dev/null +++ b/kokoro/linux/java_jdk7/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/java_jdk7/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/java_jdk7/presubmit.cfg b/kokoro/linux/java_jdk7/presubmit.cfg new file mode 100644 index 00000000..07a9df5f --- /dev/null +++ b/kokoro/linux/java_jdk7/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/java_jdk7/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/java_oracle7/build.sh b/kokoro/linux/java_oracle7/build.sh new file mode 100755 index 00000000..9ab90dad --- /dev/null +++ b/kokoro/linux/java_oracle7/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERFILE_DIR=kokoro/linux/64-bit +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="java_oracle7" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/java_oracle7/continuous.cfg b/kokoro/linux/java_oracle7/continuous.cfg new file mode 100644 index 00000000..610469b4 --- /dev/null +++ b/kokoro/linux/java_oracle7/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/java_oracle7/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/java_oracle7/presubmit.cfg b/kokoro/linux/java_oracle7/presubmit.cfg new file mode 100644 index 00000000..610469b4 --- /dev/null +++ b/kokoro/linux/java_oracle7/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/java_oracle7/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/javanano_jdk7/build.sh b/kokoro/linux/javanano_jdk7/build.sh new file mode 100755 index 00000000..2fc06475 --- /dev/null +++ b/kokoro/linux/javanano_jdk7/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERFILE_DIR=kokoro/linux/64-bit +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="javanano_jdk7" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/javanano_jdk7/continuous.cfg b/kokoro/linux/javanano_jdk7/continuous.cfg new file mode 100644 index 00000000..72936484 --- /dev/null +++ b/kokoro/linux/javanano_jdk7/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/javanano_jdk7/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/javanano_jdk7/presubmit.cfg b/kokoro/linux/javanano_jdk7/presubmit.cfg new file mode 100644 index 00000000..72936484 --- /dev/null +++ b/kokoro/linux/javanano_jdk7/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/javanano_jdk7/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/javanano_oracle7/build.sh b/kokoro/linux/javanano_oracle7/build.sh new file mode 100755 index 00000000..651d17e8 --- /dev/null +++ b/kokoro/linux/javanano_oracle7/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERFILE_DIR=kokoro/linux/64-bit +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="javanano_oracle7" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/javanano_oracle7/continuous.cfg b/kokoro/linux/javanano_oracle7/continuous.cfg new file mode 100644 index 00000000..9cac8972 --- /dev/null +++ b/kokoro/linux/javanano_oracle7/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/javanano_oracle7/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/javanano_oracle7/presubmit.cfg b/kokoro/linux/javanano_oracle7/presubmit.cfg new file mode 100644 index 00000000..9cac8972 --- /dev/null +++ b/kokoro/linux/javanano_oracle7/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/javanano_oracle7/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/javascript/build.sh b/kokoro/linux/javascript/build.sh new file mode 100755 index 00000000..c2585fb7 --- /dev/null +++ b/kokoro/linux/javascript/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERFILE_DIR=kokoro/linux/64-bit +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="javascript" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/javascript/continuous.cfg b/kokoro/linux/javascript/continuous.cfg new file mode 100644 index 00000000..b42f522f --- /dev/null +++ b/kokoro/linux/javascript/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/javascript/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/javascript/presubmit.cfg b/kokoro/linux/javascript/presubmit.cfg new file mode 100644 index 00000000..b42f522f --- /dev/null +++ b/kokoro/linux/javascript/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/javascript/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/make_test_output.py b/kokoro/linux/make_test_output.py new file mode 100644 index 00000000..98536853 --- /dev/null +++ b/kokoro/linux/make_test_output.py @@ -0,0 +1,94 @@ +"""Gathers output from test runs and create an XML file in JUnit format. + +The output files from the individual tests have been written in a directory +structure like: + + $DIR/joblog (output from "parallel --joblog joblog") + $DIR/logs/1/cpp/stdout + $DIR/logs/1/cpp/stderr + $DIR/logs/1/csharp/stdout + $DIR/logs/1/csharp/stderr + $DIR/logs/1/java_jdk7/stdout + $DIR/logs/1/java_jdk7/stderr + etc. + +This script bundles them into a single output XML file so Jenkins can show +detailed test results. It runs as the last step before the Jenkins build +finishes. +""" + +import os +import sys +from yattag import Doc +from collections import defaultdict + + +def readtests(basedir): + tests = defaultdict(dict) + + # Sample input (note: separators are tabs). + # + # Seq Host Starttime Runtime Send Receive Exitval Signal Command + # 1 : 1456263838.313 0.005 0 0 0 0 echo A + with open(basedir + "/joblog") as jobs: + firstline = next(jobs) + for line in jobs: + values = line.split("\t") + + name = values[8].split()[-1] + test = tests[name] + test["name"] = name + test["time"] = values[3] + + exitval = values[6] + if int(exitval): + # We don't have a more specific message. User should look at stderr. + test["failure"] = "TEST FAILURE" + else: + test["failure"] = False + + for testname in os.listdir(basedir + "/logs/1"): + test = tests[testname] + + with open(basedir + "/logs/1/" + testname + "/stdout") as f: + test["stdout"] = f.read() + + with open(basedir + "/logs/1/" + testname + "/stderr") as f: + test["stderr"] = f.read() + + # The cpp test is special since it doesn't run under parallel so doesn't show + # up in the job log. + tests["cpp"]["name"] = "cpp" + + with open(basedir + '/logs/1/cpp/build_time', 'r') as f: + tests["cpp"]["time"] = f.read().strip() + tests["cpp"]["failure"] = False + + ret = tests.values() + ret.sort(key=lambda x: x["name"]) + + return ret + + +def genxml(tests): + doc, tag, text = Doc().tagtext() + + with tag("testsuites"): + with tag("testsuite", name="Protobuf Tests"): + for test in tests: + with tag("testcase", name=test["name"], classname=test["name"], + time=test["time"]): + with tag("system-out"): + text(test["stdout"]) + with tag("system-err"): + text(test["stderr"]) + if test["failure"]: + with tag("failure"): + text(test["failure"]) + + return doc.getvalue() + + +sys.stderr.write("make_test_output.py: writing XML from directory: " + + sys.argv[1] + "\n") +print(genxml(readtests(sys.argv[1]))) diff --git a/kokoro/linux/php_all/build.sh b/kokoro/linux/php_all/build.sh new file mode 100755 index 00000000..e9ab84cb --- /dev/null +++ b/kokoro/linux/php_all/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERFILE_DIR=kokoro/linux/64-bit +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="php_all" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/php_all/continuous.cfg b/kokoro/linux/php_all/continuous.cfg new file mode 100644 index 00000000..dfa8037e --- /dev/null +++ b/kokoro/linux/php_all/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/php_all/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/php_all/presubmit.cfg b/kokoro/linux/php_all/presubmit.cfg new file mode 100644 index 00000000..dfa8037e --- /dev/null +++ b/kokoro/linux/php_all/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/php_all/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/pull_request_in_docker.sh b/kokoro/linux/pull_request_in_docker.sh new file mode 100755 index 00000000..23e63961 --- /dev/null +++ b/kokoro/linux/pull_request_in_docker.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# +# This is the script that runs inside Docker, once the image has been built, +# to execute all tests for the "pull request" project. + +WORKSPACE_BASE=`pwd` +MY_DIR="$(dirname "$0")" +TEST_SCRIPT=$MY_DIR/../../tests.sh +BUILD_DIR=/tmp/protobuf + +set -e # exit immediately on error +set -x # display all commands + +# The protobuf repository is mounted into our Docker image, but read-only. +# We clone into a directory inside Docker (this is faster than cp). +rm -rf $BUILD_DIR +mkdir -p $BUILD_DIR +cd $BUILD_DIR +git clone /var/local/kokoro/protobuf +cd protobuf + +# Set up the directory where our test output is going to go. +OUTPUT_DIR=`mktemp -d` +LOG_OUTPUT_DIR=$OUTPUT_DIR/logs +mkdir -p $LOG_OUTPUT_DIR/1/cpp + +################################################################################ +# cpp build needs to run first, non-parallelized, so that protoc is available +# for other builds. + +# Output filenames to follow the overall scheme used by parallel, ie: +# $DIR/logs/1/cpp/stdout +# $DIR/logs/1/cpp/stderr +# $DIR/logs/1/csharp/stdout +# $DIR/logs/1/csharp/stderr +# $DIR/logs/1/java_jdk7/stdout +# $DIR/logs/1/java_jdk7/stderr +CPP_STDOUT=$LOG_OUTPUT_DIR/1/cpp/stdout +CPP_STDERR=$LOG_OUTPUT_DIR/1/cpp/stderr + +# Time the C++ build, so we can put this info in the test output. +# It's important that we get /usr/bin/time (which supports -f and -o) and not +# the bash builtin "time" which doesn't. +TIME_CMD="/usr/bin/time -f %e -o $LOG_OUTPUT_DIR/1/cpp/build_time" + +$TIME_CMD $TEST_SCRIPT cpp > >(tee $CPP_STDOUT) 2> >(tee $CPP_STDERR >&2) + +# Other tests are run in parallel. TEST_SET is defined in +# buildcmds/pull_request{_32}.sh + +parallel --results $LOG_OUTPUT_DIR --joblog $OUTPUT_DIR/joblog $TEST_SCRIPT ::: \ + $TEST_SET \ + || true # Process test results even if tests fail. + +cat $OUTPUT_DIR/joblog + +# The directory that is copied from Docker back into the Kokoro workspace. +COPY_FROM_DOCKER=/var/local/git/protobuf/testoutput +mkdir -p $COPY_FROM_DOCKER +TESTOUTPUT_XML_FILE=$COPY_FROM_DOCKER/sponge_log.xml + +# Process all the output files from "parallel" and package them into a single +# .xml file with detailed, broken-down test output. +python $MY_DIR/make_test_output.py $OUTPUT_DIR > $TESTOUTPUT_XML_FILE + +ls -l $TESTOUTPUT_XML_FILE diff --git a/kokoro/linux/python/build.sh b/kokoro/linux/python/build.sh new file mode 100755 index 00000000..86964f55 --- /dev/null +++ b/kokoro/linux/python/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERFILE_DIR=kokoro/linux/64-bit +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="python" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/python/continuous.cfg b/kokoro/linux/python/continuous.cfg new file mode 100644 index 00000000..e2fc4136 --- /dev/null +++ b/kokoro/linux/python/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/python/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/python/presubmit.cfg b/kokoro/linux/python/presubmit.cfg new file mode 100644 index 00000000..e2fc4136 --- /dev/null +++ b/kokoro/linux/python/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/python/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/python_cpp/build.sh b/kokoro/linux/python_cpp/build.sh new file mode 100755 index 00000000..add64651 --- /dev/null +++ b/kokoro/linux/python_cpp/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERFILE_DIR=kokoro/linux/64-bit +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="python_cpp" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/python_cpp/continuous.cfg b/kokoro/linux/python_cpp/continuous.cfg new file mode 100644 index 00000000..b1b0e550 --- /dev/null +++ b/kokoro/linux/python_cpp/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/python_cpp/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/python_cpp/presubmit.cfg b/kokoro/linux/python_cpp/presubmit.cfg new file mode 100644 index 00000000..b1b0e550 --- /dev/null +++ b/kokoro/linux/python_cpp/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/python_cpp/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/ruby_all/build.sh b/kokoro/linux/ruby_all/build.sh new file mode 100755 index 00000000..ea758958 --- /dev/null +++ b/kokoro/linux/ruby_all/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERFILE_DIR=kokoro/linux/64-bit +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="ruby_all" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/ruby_all/continuous.cfg b/kokoro/linux/ruby_all/continuous.cfg new file mode 100644 index 00000000..02d76437 --- /dev/null +++ b/kokoro/linux/ruby_all/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/ruby_all/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/ruby_all/presubmit.cfg b/kokoro/linux/ruby_all/presubmit.cfg new file mode 100644 index 00000000..02d76437 --- /dev/null +++ b/kokoro/linux/ruby_all/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/ruby_all/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/objectivec/DevTools/full_mac_build.sh b/objectivec/DevTools/full_mac_build.sh index 7e1e9a05..b34e28fc 100755 --- a/objectivec/DevTools/full_mac_build.sh +++ b/objectivec/DevTools/full_mac_build.sh @@ -261,11 +261,11 @@ if [[ "${DO_XCODE_IOS_TESTS}" == "yes" ]] ; then -destination "platform=iOS Simulator,name=iPhone 7,OS=latest" # 64bit ) ;; - 9.[0-1]* ) + 9.[0-2]* ) XCODEBUILD_TEST_BASE_IOS+=( -destination "platform=iOS Simulator,name=iPhone 4s,OS=8.1" # 32bit -destination "platform=iOS Simulator,name=iPhone 7,OS=latest" # 64bit - # 9.0/9.1 both seem to often fail running destinations in parallel + # 9.0-9.2 all seem to often fail running destinations in parallel -disable-concurrent-testing ) ;; diff --git a/objectivec/DevTools/pddm_tests.py b/objectivec/DevTools/pddm_tests.py index 8a73b842..9ac6a855 100755 --- a/objectivec/DevTools/pddm_tests.py +++ b/objectivec/DevTools/pddm_tests.py @@ -311,7 +311,7 @@ foo(x, y) mc = pddm.MacroCollection(f) try: result = mc.Expand('foo(A,B)') - self.fail('Should throw exception, entry %d' % idx) + self.fail('Should throw exception! Test failed to catch recursion.') except pddm.PDDMError as e: self.assertEqual(e.message, 'Found macro recusion, invoking "foo(1, A)":\n...while expanding "bar(1, A)".\n...while expanding "foo(A,B)".') @@ -483,7 +483,7 @@ foo sf = pddm.SourceFile(f) try: sf.ProcessContent() - self.fail('Should throw exception, entry %d' % idx) + self.fail('Should throw exception! Test failed to catch macro parsing error.') except pddm.PDDMError as e: self.assertEqual(e.message, 'Attempt to redefine macro: "PDDM-DEFINE mumble(x_)"\n' @@ -503,7 +503,7 @@ foo sf = pddm.SourceFile(f) try: sf.ProcessContent() - self.fail('Should throw exception, entry %d' % idx) + self.fail('Should throw exception! Test failed to catch expand error.') except pddm.PDDMError as e: self.assertEqual(e.message, 'No macro named "foobar".\n' diff --git a/objectivec/GPBMessage.m b/objectivec/GPBMessage.m index 90485bd1..3be20b67 100644 --- a/objectivec/GPBMessage.m +++ b/objectivec/GPBMessage.m @@ -969,7 +969,8 @@ static GPBUnknownFieldSet *GetOrMakeUnknownFields(GPBMessage *self) { newValue = [value copyWithZone:zone]; } } else { - if (field.mapKeyDataType == GPBDataTypeString) { + if ((field.mapKeyDataType == GPBDataTypeString) && + GPBFieldDataTypeIsObject(field)) { // NSDictionary newValue = [value mutableCopyWithZone:zone]; } else { diff --git a/objectivec/Tests/GPBMessageTests.m b/objectivec/Tests/GPBMessageTests.m index 4d75f1e1..a3149096 100644 --- a/objectivec/Tests/GPBMessageTests.m +++ b/objectivec/Tests/GPBMessageTests.m @@ -2052,4 +2052,55 @@ XCTAssertEqual([msg1 hash], [msg1Prime hash]); } +- (void)testCopyingMapFileds { + TestMessageOfMaps *msg = [TestMessageOfMaps message]; + + msg.strToStr[@"foo"] = @"bar"; + + [msg.strToInt setInt32:1 forKey:@"mumble"]; + [msg.intToStr setObject:@"wee" forKey:42]; + [msg.intToInt setInt32:123 forKey:321]; + + [msg.strToBool setBool:YES forKey:@"one"]; + [msg.boolToStr setObject:@"something" forKey:YES]; + [msg.boolToBool setBool:YES forKey:NO]; + + [msg.intToBool setBool:YES forKey:13]; + [msg.boolToInt setInt32:111 forKey:NO]; + + TestAllTypes *subMsg1 = [TestAllTypes message]; + subMsg1.optionalInt32 = 1; + TestAllTypes *subMsg2 = [TestAllTypes message]; + subMsg1.optionalInt32 = 2; + TestAllTypes *subMsg3 = [TestAllTypes message]; + subMsg1.optionalInt32 = 3; + + msg.strToMsg[@"baz"] = subMsg1; + [msg.intToMsg setObject:subMsg2 forKey:222]; + [msg.boolToMsg setObject:subMsg3 forKey:YES]; + + TestMessageOfMaps *msg2 = [[msg copy] autorelease]; + XCTAssertNotNil(msg2); + XCTAssertEqualObjects(msg2, msg); + XCTAssertTrue(msg2 != msg); // ptr compare + XCTAssertTrue(msg.strToStr != msg2.strToStr); // ptr compare + XCTAssertTrue(msg.intToStr != msg2.intToStr); // ptr compare + XCTAssertTrue(msg.intToInt != msg2.intToInt); // ptr compare + XCTAssertTrue(msg.strToBool != msg2.strToBool); // ptr compare + XCTAssertTrue(msg.boolToStr != msg2.boolToStr); // ptr compare + XCTAssertTrue(msg.boolToBool != msg2.boolToBool); // ptr compare + XCTAssertTrue(msg.intToBool != msg2.intToBool); // ptr compare + XCTAssertTrue(msg.boolToInt != msg2.boolToInt); // ptr compare + XCTAssertTrue(msg.strToMsg != msg2.strToMsg); // ptr compare + XCTAssertTrue(msg.intToMsg != msg2.intToMsg); // ptr compare + XCTAssertTrue(msg.boolToMsg != msg2.boolToMsg); // ptr compare + + XCTAssertTrue(msg.strToMsg[@"baz"] != msg2.strToMsg[@"baz"]); // ptr compare + XCTAssertEqualObjects(msg.strToMsg[@"baz"], msg2.strToMsg[@"baz"]); + XCTAssertTrue([msg.intToMsg objectForKey:222] != [msg2.intToMsg objectForKey:222]); // ptr compare + XCTAssertEqualObjects([msg.intToMsg objectForKey:222], [msg2.intToMsg objectForKey:222]); + XCTAssertTrue([msg.boolToMsg objectForKey:YES] != [msg2.boolToMsg objectForKey:YES]); // ptr compare + XCTAssertEqualObjects([msg.boolToMsg objectForKey:YES], [msg2.boolToMsg objectForKey:YES]); +} + @end diff --git a/objectivec/Tests/unittest_objc.proto b/objectivec/Tests/unittest_objc.proto index e5577faf..b0eb4723 100644 --- a/objectivec/Tests/unittest_objc.proto +++ b/objectivec/Tests/unittest_objc.proto @@ -58,6 +58,27 @@ message TestRecursiveMessageWithRepeatedField { map<string, string> str_to_str = 5; } +// Message with a few types of maps to cover the different custom flows +// in the runtime. +message TestMessageOfMaps { + map<string, string> str_to_str = 1; + + map<string, int32> str_to_int = 2; + map<int32, string> int_to_str = 3; + map<int32, int32> int_to_int = 4; + + map<string, bool> str_to_bool = 5; + map<bool, string> bool_to_str = 6; + map<bool, bool> bool_to_bool = 7; + + map<int32, bool> int_to_bool = 8; + map<bool, int32> bool_to_int = 9; + + map<string, TestAllTypes> str_to_msg = 10; + map<int32, TestAllTypes> int_to_msg = 11; + map<bool, TestAllTypes> bool_to_msg = 12; +} + // Recursive message and extension to for testing autocreators at different // depths. message TestRecursiveExtension { diff --git a/python/google/protobuf/internal/containers.py b/python/google/protobuf/internal/containers.py index 68be9e54..c6a3692a 100755 --- a/python/google/protobuf/internal/containers.py +++ b/python/google/protobuf/internal/containers.py @@ -549,10 +549,10 @@ class MessageMap(MutableMapping): self._values = {} def __getitem__(self, key): + key = self._key_checker.CheckValue(key) try: return self._values[key] except KeyError: - key = self._key_checker.CheckValue(key) new_element = self._message_descriptor._concrete_class() new_element._SetListener(self._message_listener) self._values[key] = new_element @@ -584,12 +584,14 @@ class MessageMap(MutableMapping): return default def __contains__(self, item): + item = self._key_checker.CheckValue(item) return item in self._values def __setitem__(self, key, value): raise ValueError('May not set values directly, call my_map[key].foo = 5') def __delitem__(self, key): + key = self._key_checker.CheckValue(key) del self._values[key] self._message_listener.Modified() diff --git a/python/google/protobuf/internal/descriptor_test.py b/python/google/protobuf/internal/descriptor_test.py index f1d5934e..02a43d15 100755 --- a/python/google/protobuf/internal/descriptor_test.py +++ b/python/google/protobuf/internal/descriptor_test.py @@ -742,6 +742,19 @@ class DescriptorCopyToProtoTest(unittest.TestCase): deprecated: true > > + field { + name: "deprecated_int32_in_oneof" + number: 2 + label: LABEL_OPTIONAL + type: TYPE_INT32 + options { + deprecated: true + } + oneof_index: 0 + } + oneof_decl { + name: "oneof_fields" + } """ self._InternalTestCopyToProto( diff --git a/python/google/protobuf/internal/message_test.py b/python/google/protobuf/internal/message_test.py index a303b1aa..8dae6377 100755 --- a/python/google/protobuf/internal/message_test.py +++ b/python/google/protobuf/internal/message_test.py @@ -1480,12 +1480,8 @@ class Proto3Test(BaseTestCase): submsg = msg.map_int32_foreign_message[5] self.assertIs(submsg, msg.map_int32_foreign_message.get(5)) - # TODO(jieluo): Fix python and cpp extension diff. - if api_implementation.Type() == 'cpp': - with self.assertRaises(TypeError): - msg.map_int32_foreign_message.get('') - else: - self.assertEqual(None, msg.map_int32_foreign_message.get('')) + with self.assertRaises(TypeError): + msg.map_int32_foreign_message.get('') def testScalarMap(self): msg = map_unittest_pb2.TestMap() @@ -1695,12 +1691,8 @@ class Proto3Test(BaseTestCase): del msg2.map_int32_foreign_message[222] self.assertFalse(222 in msg2.map_int32_foreign_message) - if api_implementation.Type() == 'cpp': - with self.assertRaises(TypeError): - del msg2.map_int32_foreign_message[''] - else: - with self.assertRaises(KeyError): - del msg2.map_int32_foreign_message[''] + with self.assertRaises(TypeError): + del msg2.map_int32_foreign_message[''] def testMergeFromBadType(self): msg = map_unittest_pb2.TestMap() diff --git a/python/release.sh b/python/release.sh index b0bf3b3f..01fc5e23 100755 --- a/python/release.sh +++ b/python/release.sh @@ -47,6 +47,7 @@ DEV=$2 # Make sure all files are world-readable. find python -type d -exec chmod a+r,a+x {} + find python -type f -exec chmod a+r {} + +umask 0022 # Check that the supplied version number matches what's inside the source code. SOURCE_VERSION=`get_source_version` diff --git a/src/google/protobuf/compiler/java/java_enum_field.cc b/src/google/protobuf/compiler/java/java_enum_field.cc index 9f7bb349..642cdd36 100644 --- a/src/google/protobuf/compiler/java/java_enum_field.cc +++ b/src/google/protobuf/compiler/java/java_enum_field.cc @@ -80,7 +80,7 @@ void SetEnumVariables(const FieldDescriptor* descriptor, // Use deprecated valueOf() method to be compatible with old generated code // for v2.5.0/v2.6.1. // TODO(xiaofeng): Use "forNumber" when we no longer support compatibility - // with v2.5.0/v2.6.1. + // with v2.5.0/v2.6.1, and remove the @SuppressWarnings annotations. (*variables)["for_number"] = "valueOf"; if (SupportFieldPresence(descriptor->file())) { @@ -199,6 +199,7 @@ GenerateMembers(io::Printer* printer) const { WriteFieldDocComment(printer, descriptor_); printer->Print(variables_, "$deprecation$public $type$ ${$get$capitalized_name$$}$() {\n" + " @SuppressWarnings(\"deprecation\")\n" " $type$ result = $type$.$for_number$($name$_);\n" " return result == null ? $unknown$ : result;\n" "}\n"); @@ -237,6 +238,7 @@ GenerateBuilderMembers(io::Printer* printer) const { WriteFieldDocComment(printer, descriptor_); printer->Print(variables_, "$deprecation$public $type$ ${$get$capitalized_name$$}$() {\n" + " @SuppressWarnings(\"deprecation\")\n" " $type$ result = $type$.$for_number$($name$_);\n" " return result == null ? $unknown$ : result;\n" "}\n"); @@ -320,6 +322,7 @@ GenerateParsingCode(io::Printer* printer) const { } else { printer->Print(variables_, "int rawValue = input.readEnum();\n" + " @SuppressWarnings(\"deprecation\")\n" "$type$ value = $type$.$for_number$(rawValue);\n" "if (value == null) {\n" " unknownFields.mergeVarintField($number$, rawValue);\n" @@ -412,6 +415,7 @@ GenerateMembers(io::Printer* printer) const { printer->Print(variables_, "$deprecation$public $type$ ${$get$capitalized_name$$}$() {\n" " if ($has_oneof_case_message$) {\n" + " @SuppressWarnings(\"deprecation\")\n" " $type$ result = $type$.$for_number$(\n" " (java.lang.Integer) $oneof_name$_);\n" " return result == null ? $unknown$ : result;\n" @@ -456,6 +460,7 @@ GenerateBuilderMembers(io::Printer* printer) const { printer->Print(variables_, "$deprecation$public $type$ ${$get$capitalized_name$$}$() {\n" " if ($has_oneof_case_message$) {\n" + " @SuppressWarnings(\"deprecation\")\n" " $type$ result = $type$.$for_number$(\n" " (java.lang.Integer) $oneof_name$_);\n" " return result == null ? $unknown$ : result;\n" @@ -517,6 +522,7 @@ GenerateParsingCode(io::Printer* printer) const { } else { printer->Print(variables_, "int rawValue = input.readEnum();\n" + "@SuppressWarnings(\"deprecation\")\n" "$type$ value = $type$.$for_number$(rawValue);\n" "if (value == null) {\n" " unknownFields.mergeVarintField($number$, rawValue);\n" @@ -626,6 +632,7 @@ GenerateMembers(io::Printer* printer) const { " new com.google.protobuf.Internal.ListAdapter.Converter<\n" " java.lang.Integer, $type$>() {\n" " public $type$ convert(java.lang.Integer from) {\n" + " @SuppressWarnings(\"deprecation\")\n" " $type$ result = $type$.$for_number$(from);\n" " return result == null ? $unknown$ : result;\n" " }\n" @@ -879,6 +886,7 @@ GenerateParsingCode(io::Printer* printer) const { } else { printer->Print(variables_, "int rawValue = input.readEnum();\n" + "@SuppressWarnings(\"deprecation\")\n" "$type$ value = $type$.$for_number$(rawValue);\n" "if (value == null) {\n" " unknownFields.mergeVarintField($number$, rawValue);\n" diff --git a/src/google/protobuf/compiler/java/java_message.cc b/src/google/protobuf/compiler/java/java_message.cc index f2ff2800..eca7cf9a 100644 --- a/src/google/protobuf/compiler/java/java_message.cc +++ b/src/google/protobuf/compiler/java/java_message.cc @@ -437,11 +437,10 @@ void ImmutableMessageGenerator::Generate(io::Printer* printer) { for (int j = 0; j < descriptor_->oneof_decl(i)->field_count(); j++) { const FieldDescriptor* field = descriptor_->oneof_decl(i)->field(j); printer->Print( - "$field_name$($field_number$),\n", - "field_name", - ToUpper(field->name()), - "field_number", - SimpleItoa(field->number())); + "$deprecation$$field_name$($field_number$),\n", + "deprecation", field->options().deprecated() ? "@java.lang.Deprecated " : "", + "field_name", ToUpper(field->name()), + "field_number", SimpleItoa(field->number())); } printer->Print( "$cap_oneof_name$_NOT_SET(0);\n", diff --git a/src/google/protobuf/stubs/platform_macros.h b/src/google/protobuf/stubs/platform_macros.h index 7a54060d..c3a64dd2 100644 --- a/src/google/protobuf/stubs/platform_macros.h +++ b/src/google/protobuf/stubs/platform_macros.h @@ -50,6 +50,9 @@ #elif defined(_M_ARM) || defined(__ARMEL__) #define GOOGLE_PROTOBUF_ARCH_ARM 1 #define GOOGLE_PROTOBUF_ARCH_32_BIT 1 +#elif defined(_M_ARM64) +#define GOOGLE_PROTOBUF_ARCH_ARM 1 +#define GOOGLE_PROTOBUF_ARCH_64_BIT 1 #elif defined(__aarch64__) #define GOOGLE_PROTOBUF_ARCH_AARCH64 1 #define GOOGLE_PROTOBUF_ARCH_64_BIT 1 diff --git a/src/google/protobuf/unittest.proto b/src/google/protobuf/unittest.proto index 45a0edad..2a288daa 100644 --- a/src/google/protobuf/unittest.proto +++ b/src/google/protobuf/unittest.proto @@ -189,6 +189,9 @@ message NestedTestAllTypes { message TestDeprecatedFields { optional int32 deprecated_int32 = 1 [deprecated=true]; + oneof oneof_fields { + int32 deprecated_int32_in_oneof = 2 [deprecated=true]; + } } message TestDeprecatedMessage { diff --git a/src/google/protobuf/util/json_util.cc b/src/google/protobuf/util/json_util.cc index ce3569ce..25e78a65 100644 --- a/src/google/protobuf/util/json_util.cc +++ b/src/google/protobuf/util/json_util.cc @@ -50,7 +50,9 @@ namespace util { namespace internal { ZeroCopyStreamByteSink::~ZeroCopyStreamByteSink() { - stream_->BackUp(buffer_size_); + if (buffer_size_ > 0) { + stream_->BackUp(buffer_size_); + } } void ZeroCopyStreamByteSink::Append(const char* bytes, size_t len) { diff --git a/src/google/protobuf/util/json_util_test.cc b/src/google/protobuf/util/json_util_test.cc index 25c7e96c..5b714bb1 100644 --- a/src/google/protobuf/util/json_util_test.cc +++ b/src/google/protobuf/util/json_util_test.cc @@ -40,6 +40,7 @@ #include <google/protobuf/util/json_format_proto3.pb.h> #include <google/protobuf/util/type_resolver.h> #include <google/protobuf/util/type_resolver_util.h> +#include <google/protobuf/io/zero_copy_stream_impl.h> #include <gtest/gtest.h> namespace google { @@ -457,6 +458,29 @@ TEST(ZeroCopyStreamByteSinkTest, TestAllInputOutputPatterns) { } } +TEST_F(JsonUtilTest, TestWrongJsonInput) { + using namespace google::protobuf; + const char json[] = "{\"unknown_field\":\"some_value\"}"; + io::ArrayInputStream input_stream(json, strlen(json)); + char proto_buffer[10000]; + io::ArrayOutputStream output_stream(proto_buffer, sizeof(proto_buffer)); + std::string message_type = "type.googleapis.com/proto3.TestMessage"; + TypeResolver* resolver = NewTypeResolverForDescriptorPool( + "type.googleapis.com", + DescriptorPool::generated_pool()); + + util::Status result_status = util::JsonToBinaryStream(resolver, + message_type, + &input_stream, + &output_stream); + + delete resolver; + + EXPECT_FALSE(result_status.ok()); + EXPECT_EQ(result_status.error_code(), + google::protobuf::util::error::INVALID_ARGUMENT); +} + } // namespace } // namespace util } // namespace protobuf |