10 files changed, 95 insertions, 94 deletions
diff --git a/kokoro/linux/bazel/build.sh b/kokoro/linux/bazel/build.sh
index e0c1ee42..1769301d 100755
--- a/kokoro/linux/bazel/build.sh
+++ b/kokoro/linux/bazel/build.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 #
 # Build file to set up and run tests
+set -ex
 
 # Change to repo root
 cd $(dirname $0)/../../..
diff --git a/kokoro/linux/benchmark/build.sh b/kokoro/linux/benchmark/build.sh
index 750c3e5c..527485c8 100755
--- a/kokoro/linux/benchmark/build.sh
+++ b/kokoro/linux/benchmark/build.sh
@@ -27,8 +27,8 @@ cd $oldpwd
 ./configure CXXFLAGS="-fPIC -O2"
 make -j8
 cd python
-python setup.py -q build --cpp_implementation
-pip install .
+python setup.py build --cpp_implementation
+pip install . --user
 
 
 # build and run Python benchmark
@@ -41,10 +41,10 @@ echo "benchmarking pure python..."
 ./python-pure-python-benchmark --json --behavior_prefix="pure-python-benchmark" $datasets  >> tmp/python_result.json
 echo "," >> "tmp/python_result.json"
 echo "benchmarking python cpp reflection..."
-env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" ./python-cpp-reflection-benchmark --json --behavior_prefix="cpp-reflection-benchmark" $datasets  >> tmp/python_result.json
+env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-reflection-benchmark --json --behavior_prefix="cpp-reflection-benchmark" $datasets  >> tmp/python_result.json
 echo "," >> "tmp/python_result.json"
 echo "benchmarking python cpp generated code..."
-env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" ./python-cpp-generated-code-benchmark --json --behavior_prefix="cpp-generated-code-benchmark" $datasets >> tmp/python_result.json
+env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-generated-code-benchmark --json --behavior_prefix="cpp-generated-code-benchmark" $datasets >> tmp/python_result.json
 echo "]" >> "tmp/python_result.json"
 cd $oldpwd
 
@@ -52,6 +52,11 @@ cd $oldpwd
 ./configure
 make clean && make -j8
 
+# build Java protobuf
+cd java
+mvn package
+cd ..
+
 # build CPP benchmark
 cd benchmarks
 mv tmp/python_result.json . && make clean && make -j8 cpp-benchmark && mv python_result.json tmp
@@ -63,8 +68,8 @@ cd $oldpwd
 export PATH="`pwd`/src:$PATH"
 export GOPATH="$HOME/gocode"
 mkdir -p "$GOPATH/src/github.com/google"
-rm -f "$GOPATH/src/github.com/google/protobuf"
-ln -s "`pwd`" "$GOPATH/src/github.com/google/protobuf"
+rm -f "$GOPATH/src/github.com/protocolbuffers/protobuf"
+ln -s "`pwd`" "$GOPATH/src/github.com/protocolbuffers/protobuf"
 export PATH="$GOPATH/bin:$PATH"
 go get github.com/golang/protobuf/protoc-gen-go
 
@@ -81,7 +86,7 @@ echo "benchmarking java..."
 
 # upload result to bq
 make python_add_init
-python util/run_and_upload.py -cpp="../tmp/cpp_result.json" -java="../tmp/java_result.json" \
+env LD_LIBRARY_PATH="$oldpwd/src/.libs" python -m util.result_uploader -cpp="../tmp/cpp_result.json" -java="../tmp/java_result.json" \
     -python="../tmp/python_result.json" -go="../tmp/go_result.txt"
 
 cd $oldpwd
diff --git a/kokoro/linux/build_and_run_docker.sh b/kokoro/linux/build_and_run_docker.sh
index f9c60710..b81a3684 100755
--- a/kokoro/linux/build_and_run_docker.sh
+++ b/kokoro/linux/build_and_run_docker.sh
@@ -8,7 +8,6 @@
 # OUTPUT_DIR - Directory that will be copied from inside docker after finishing.
 # $@ - Extra args to pass to docker run
 
-
 set -ex
 
 cd $(dirname $0)/../..
@@ -39,7 +38,6 @@ docker run \
   -e KOKORO_BUILD_ID=$KOKORO_BUILD_ID \
   -e EXTERNAL_GIT_ROOT="/var/local/kokoro/protobuf" \
   -e TEST_SET="$TEST_SET" \
-  -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \
   -v "$git_root:/var/local/kokoro/protobuf:ro" \
   -v $CCACHE_DIR:$CCACHE_DIR \
   -w /var/local/git/protobuf \
@@ -47,16 +45,9 @@ docker run \
   $DOCKER_IMAGE_NAME \
   bash -l "/var/local/kokoro/protobuf/$DOCKER_RUN_SCRIPT" || FAILED="true"
 
-# Copy output artifacts
-if [ "$OUTPUT_DIR" != "" ]
-then
-  docker cp "$CONTAINER_NAME:/var/local/git/protobuf/$OUTPUT_DIR" "${git_root}/kokoro" || FAILED="true"
-fi
-
 # remove the container, possibly killing it first
 docker rm -f $CONTAINER_NAME || true
 
-if [ "$FAILED" != "" ]
-then
+[ -z "$FAILED" ] || {
   exit 1
-fi
+}
diff --git a/kokoro/linux/pull_request_in_docker.sh b/kokoro/linux/pull_request_in_docker.sh
index e28ef958..a048c470 100755
--- a/kokoro/linux/pull_request_in_docker.sh
+++ b/kokoro/linux/pull_request_in_docker.sh
@@ -22,52 +22,4 @@ cd protobuf
 # Initialize any submodules:
 git submodule update --init --recursive
 
-# Set up the directory where our test output is going to go.
-OUTPUT_DIR=`mktemp -d`
-LOG_OUTPUT_DIR=$OUTPUT_DIR/logs
-mkdir -p $LOG_OUTPUT_DIR/1/cpp
-
-################################################################################
-# cpp build needs to run first, non-parallelized, so that protoc is available
-# for other builds.
-
-# Output filenames to follow the overall scheme used by parallel, ie:
-#  $DIR/logs/1/cpp/stdout
-#  $DIR/logs/1/cpp/stderr
-#  $DIR/logs/1/csharp/stdout
-#  $DIR/logs/1/csharp/stderr
-#  $DIR/logs/1/java_jdk7/stdout
-#  $DIR/logs/1/java_jdk7/stderr
-CPP_STDOUT=$LOG_OUTPUT_DIR/1/cpp/stdout
-CPP_STDERR=$LOG_OUTPUT_DIR/1/cpp/stderr
-
-# Time the C++ build, so we can put this info in the test output.
-# It's important that we get /usr/bin/time (which supports -f and -o) and not
-# the bash builtin "time" which doesn't.
-TIME_CMD="/usr/bin/time -f %e -o $LOG_OUTPUT_DIR/1/cpp/build_time"
-
-$TIME_CMD $TEST_SCRIPT cpp > >(tee $CPP_STDOUT) 2> >(tee $CPP_STDERR >&2)
-
-# Other tests are run in parallel. TEST_SET is defined in
-# buildcmds/pull_request{_32}.sh
-
-parallel --results $LOG_OUTPUT_DIR --joblog $OUTPUT_DIR/joblog $TEST_SCRIPT ::: \
-  $TEST_SET \
-  || FAILED="true"  # Process test results even if tests fail.
-
-cat $OUTPUT_DIR/joblog
-
-# The directory that is copied from Docker back into the Kokoro workspace.
-COPY_FROM_DOCKER=/var/local/git/protobuf/testoutput
-mkdir -p $COPY_FROM_DOCKER
-TESTOUTPUT_XML_FILE=$COPY_FROM_DOCKER/sponge_log.xml
-
-# Process all the output files from "parallel" and package them into a single
-# .xml file with detailed, broken-down test output.
-python $MY_DIR/make_test_output.py $OUTPUT_DIR > $TESTOUTPUT_XML_FILE
-
-ls -l $TESTOUTPUT_XML_FILE
-
-if [ "$FAILED" == "true" ]; then
-	exit 1
-fi
+$TEST_SCRIPT $TEST_SET
diff --git a/kokoro/macos/jruby/build.sh b/kokoro/macos/jruby/build.sh
deleted file mode 100755
index c82eaebf..00000000
--- a/kokoro/macos/jruby/build.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-#
-# Build file to set up and run tests
-
-# Change to repo root
-cd $(dirname $0)/../../..
-
-# Prepare worker environment to run tests
-source kokoro/macos/prepare_build_macos_rc
-
-./tests.sh jruby
diff --git a/kokoro/macos/jruby/continuous.cfg b/kokoro/macos/jruby/continuous.cfg
deleted file mode 100644
index f1310fd3..00000000
--- a/kokoro/macos/jruby/continuous.cfg
+++ /dev/null
@@ -1,5 +0,0 @@
-# Config file for running tests in Kokoro
-
-# Location of the build script in repository
-build_file: "protobuf/kokoro/macos/jruby/build.sh"
-timeout_mins: 1440
diff --git a/kokoro/macos/jruby/presubmit.cfg b/kokoro/macos/jruby/presubmit.cfg
deleted file mode 100644
index f1310fd3..00000000
--- a/kokoro/macos/jruby/presubmit.cfg
+++ /dev/null
@@ -1,5 +0,0 @@
-# Config file for running tests in Kokoro
-
-# Location of the build script in repository
-build_file: "protobuf/kokoro/macos/jruby/build.sh"
-timeout_mins: 1440
diff --git a/kokoro/release/collect_all_artifacts.cfg b/kokoro/release/collect_all_artifacts.cfg
new file mode 100644
index 00000000..3da1a7c4
--- /dev/null
+++ b/kokoro/release/collect_all_artifacts.cfg
@@ -0,0 +1,7 @@
+build_file: "protobuf/kokoro/release/collect_all_artifacts.sh"
+
+action {
+  define_artifacts {
+    regex: "github/protobuf/artifacts/**"
+  }
+}
diff --git a/kokoro/release/collect_all_artifacts.sh b/kokoro/release/collect_all_artifacts.sh
new file mode 100755
index 00000000..0023937d
--- /dev/null
+++ b/kokoro/release/collect_all_artifacts.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+set -ex
+
+# Change to repo root.
+cd $(dirname $0)/../..
+
+# Initialize any submodules.
+git submodule update --init --recursive
+
+# The directory with all resulting artifacts
+mkdir -p artifacts
+
+# Artifacts from all predecessor jobs get copied to this directory by kokoro
+INPUT_ARTIFACTS_DIR="${KOKORO_GFILE_DIR}/github/protobuf"
+
+# TODO(jtattermusch): remove listing the files, but for now it make it easier
+# to iterate on the script.
+ls -R ${INPUT_ARTIFACTS_DIR}
+
+# ====================================
+# Copy to expose all the artifacts from the predecessor jobs to the output
+# TODO(jtattermusch): the directory layout of the artifact builds is pretty messy,
+# so will be the output artifacts of this job.
+cp -r ${INPUT_ARTIFACTS_DIR}/* artifacts
+
+# ====================================
+# Build Google.Protobuf.Tools C# nuget
+# The reason it's being done in this script is that we need access to protoc binaries
+# built on multiple platform (the build is performed by the "build artifact" step)
+# and adding and extra chained build just for building the Google.Protobuf.Tools
+# nuget seems like an overkill.
+cd csharp
+mkdir -p protoc/windows_x86
+mkdir -p protoc/windows_x64
+cp ${INPUT_ARTIFACTS_DIR}/build32/Release/protoc.exe protoc/windows_x86/protoc.exe
+cp ${INPUT_ARTIFACTS_DIR}/build64/Release/protoc.exe protoc/windows_x64/protoc.exe
+
+mkdir -p protoc/linux_x86
+mkdir -p protoc/linux_x64
+# Because of maven unrelated reasonse the linux protoc binaries have a dummy .exe extension.
+# For the Google.Protobuf.Tools nuget, we don't want that expection, so we just remove it.
+cp ${INPUT_ARTIFACTS_DIR}/protoc-artifacts/target/linux/x86_32/protoc.exe protoc/linux_x86/protoc
+cp ${INPUT_ARTIFACTS_DIR}/protoc-artifacts/target/linux/x86_64/protoc.exe protoc/linux_x64/protoc
+
+mkdir -p protoc/macosx_x86
+mkdir -p protoc/macosx_x64
+cp ${INPUT_ARTIFACTS_DIR}/build32/src/protoc protoc/macosx_x86/protoc
+cp ${INPUT_ARTIFACTS_DIR}/build64/src/protoc protoc/macosx_x64/protoc
+
+# Install nuget (will also install  mono)
+# TODO(jtattermusch): use "mono:5.14" docker image instead so we don't have to apt-get install
+sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
+sudo apt install apt-transport-https
+echo "deb https://download.mono-project.com/repo/ubuntu stable-trusty main" | sudo tee /etc/apt/sources.list.d/mono-official-stable.list
+sudo apt update
+sudo apt-get install -y nuget
+
+nuget pack Google.Protobuf.Tools.nuspec
+
+# Copy the nupkg to the output artifacts
+cp Google.Protobuf.Tools.*.nupkg ../artifacts
diff --git a/kokoro/release/protoc/linux/build.sh b/kokoro/release/protoc/linux/build.sh
index 6c7049d7..f0cde494 100755
--- a/kokoro/release/protoc/linux/build.sh
+++ b/kokoro/release/protoc/linux/build.sh
@@ -11,6 +11,16 @@ git submodule update --init --recursive
 # Generate the configure script.
 ./autogen.sh
 
+# Cross-build for aarch64 and ppc64le. Note: we do these builds first to avoid
+# file permission issues. The Docker builds will create directories owned by
+# root, which causes problems if we try to add new artifacts to those
+# directories afterward.
+sudo apt install -y g++-aarch64-linux-gnu
+protoc-artifacts/build-protoc.sh linux aarch_64 protoc
+
+sudo apt install -y g++-powerpc64le-linux-gnu
+protoc-artifacts/build-protoc.sh linux ppcle_64 protoc
+
 # Use docker image to build linux artifacts.
 DOCKER_IMAGE_NAME=protobuf/protoc_$(sha1sum protoc-artifacts/Dockerfile | cut -f1 -d " ")
 docker pull $DOCKER_IMAGE_NAME
@@ -28,9 +38,3 @@ docker run -v $(pwd):/var/local/protobuf --rm $DOCKER_IMAGE_NAME \
   echo "Failed to build protoc for linux + x86_32."
   exit 1
 }
-
-# Cross-build for some architectures.
-# TODO(xiaofeng): It currently fails with "machine `aarch64' not recognized"
-# error.
-# sudo apt install -y g++-aarch64-linux-gnu
-# protoc-artifacts/build-protoc.sh linux aarch_64 protoc