diff options
Diffstat (limited to 'kokoro/release')
24 files changed, 430 insertions, 19 deletions
diff --git a/kokoro/release/collect_all_artifacts.cfg b/kokoro/release/collect_all_artifacts.cfg new file mode 100644 index 00000000..3da1a7c4 --- /dev/null +++ b/kokoro/release/collect_all_artifacts.cfg @@ -0,0 +1,7 @@ +build_file: "protobuf/kokoro/release/collect_all_artifacts.sh" + +action { + define_artifacts { + regex: "github/protobuf/artifacts/**" + } +} diff --git a/kokoro/release/collect_all_artifacts.sh b/kokoro/release/collect_all_artifacts.sh new file mode 100755 index 00000000..0023937d --- /dev/null +++ b/kokoro/release/collect_all_artifacts.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +set -ex + +# Change to repo root. +cd $(dirname $0)/../.. + +# Initialize any submodules. +git submodule update --init --recursive + +# The directory with all resulting artifacts +mkdir -p artifacts + +# Artifacts from all predecessor jobs get copied to this directory by kokoro +INPUT_ARTIFACTS_DIR="${KOKORO_GFILE_DIR}/github/protobuf" + +# TODO(jtattermusch): remove listing the files, but for now it make it easier +# to iterate on the script. +ls -R ${INPUT_ARTIFACTS_DIR} + +# ==================================== +# Copy to expose all the artifacts from the predecessor jobs to the output +# TODO(jtattermusch): the directory layout of the artifact builds is pretty messy, +# so will be the output artifacts of this job. +cp -r ${INPUT_ARTIFACTS_DIR}/* artifacts + +# ==================================== +# Build Google.Protobuf.Tools C# nuget +# The reason it's being done in this script is that we need access to protoc binaries +# built on multiple platform (the build is performed by the "build artifact" step) +# and adding and extra chained build just for building the Google.Protobuf.Tools +# nuget seems like an overkill. +cd csharp +mkdir -p protoc/windows_x86 +mkdir -p protoc/windows_x64 +cp ${INPUT_ARTIFACTS_DIR}/build32/Release/protoc.exe protoc/windows_x86/protoc.exe +cp ${INPUT_ARTIFACTS_DIR}/build64/Release/protoc.exe protoc/windows_x64/protoc.exe + +mkdir -p protoc/linux_x86 +mkdir -p protoc/linux_x64 +# Because of maven unrelated reasonse the linux protoc binaries have a dummy .exe extension. +# For the Google.Protobuf.Tools nuget, we don't want that expection, so we just remove it. +cp ${INPUT_ARTIFACTS_DIR}/protoc-artifacts/target/linux/x86_32/protoc.exe protoc/linux_x86/protoc +cp ${INPUT_ARTIFACTS_DIR}/protoc-artifacts/target/linux/x86_64/protoc.exe protoc/linux_x64/protoc + +mkdir -p protoc/macosx_x86 +mkdir -p protoc/macosx_x64 +cp ${INPUT_ARTIFACTS_DIR}/build32/src/protoc protoc/macosx_x86/protoc +cp ${INPUT_ARTIFACTS_DIR}/build64/src/protoc protoc/macosx_x64/protoc + +# Install nuget (will also install mono) +# TODO(jtattermusch): use "mono:5.14" docker image instead so we don't have to apt-get install +sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF +sudo apt install apt-transport-https +echo "deb https://download.mono-project.com/repo/ubuntu stable-trusty main" | sudo tee /etc/apt/sources.list.d/mono-official-stable.list +sudo apt update +sudo apt-get install -y nuget + +nuget pack Google.Protobuf.Tools.nuspec + +# Copy the nupkg to the output artifacts +cp Google.Protobuf.Tools.*.nupkg ../artifacts diff --git a/kokoro/release/csharp/windows/build_nuget.bat b/kokoro/release/csharp/windows/build_nuget.bat new file mode 100644 index 00000000..5da1e136 --- /dev/null +++ b/kokoro/release/csharp/windows/build_nuget.bat @@ -0,0 +1,5 @@ +@rem enter repo root +cd /d %~dp0\..\..\..\.. + +cd csharp +call build_packages.bat diff --git a/kokoro/release/csharp/windows/release.cfg b/kokoro/release/csharp/windows/release.cfg new file mode 100644 index 00000000..f508c65b --- /dev/null +++ b/kokoro/release/csharp/windows/release.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/release/csharp/windows/build_nuget.bat" +timeout_mins: 60 + +action { + define_artifacts { + regex: "**/*.nupkg" + } +} diff --git a/kokoro/release/linux/build_artifacts.sh b/kokoro/release/linux/build_artifacts.sh deleted file mode 100755 index ae7dc813..00000000 --- a/kokoro/release/linux/build_artifacts.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -set -ex - -# change to repo root -cd $(dirname $0)/../../.. - -source kokoro/release/linux/prepare_build.sh - -# ruby environment -source kokoro/release/linux/ruby/ruby_build_environment.sh - -# build artifacts -bash kokoro/release/linux/ruby/ruby_build.sh diff --git a/kokoro/release/protoc/linux/build.sh b/kokoro/release/protoc/linux/build.sh new file mode 100755 index 00000000..f0cde494 --- /dev/null +++ b/kokoro/release/protoc/linux/build.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -x + +# Change to repo root. +cd $(dirname $0)/../../../.. + +# Initialize any submodules. +git submodule update --init --recursive + +# Generate the configure script. +./autogen.sh + +# Cross-build for aarch64 and ppc64le. Note: we do these builds first to avoid +# file permission issues. The Docker builds will create directories owned by +# root, which causes problems if we try to add new artifacts to those +# directories afterward. +sudo apt install -y g++-aarch64-linux-gnu +protoc-artifacts/build-protoc.sh linux aarch_64 protoc + +sudo apt install -y g++-powerpc64le-linux-gnu +protoc-artifacts/build-protoc.sh linux ppcle_64 protoc + +# Use docker image to build linux artifacts. +DOCKER_IMAGE_NAME=protobuf/protoc_$(sha1sum protoc-artifacts/Dockerfile | cut -f1 -d " ") +docker pull $DOCKER_IMAGE_NAME + +docker run -v $(pwd):/var/local/protobuf --rm $DOCKER_IMAGE_NAME \ + bash -l /var/local/protobuf/protoc-artifacts/build-protoc.sh \ + linux x86_64 protoc || { + echo "Failed to build protoc for linux + x86_64." + exit 1 +} + +docker run -v $(pwd):/var/local/protobuf --rm $DOCKER_IMAGE_NAME \ + bash -l /var/local/protobuf/protoc-artifacts/build-protoc.sh \ + linux x86_32 protoc || { + echo "Failed to build protoc for linux + x86_32." + exit 1 +} diff --git a/kokoro/release/protoc/linux/release.cfg b/kokoro/release/protoc/linux/release.cfg new file mode 100644 index 00000000..4d8b7b61 --- /dev/null +++ b/kokoro/release/protoc/linux/release.cfg @@ -0,0 +1,7 @@ +build_file: "protobuf/kokoro/release/protoc/linux/build.sh" + +action { + define_artifacts { + regex: "**/protoc.exe" + } +} diff --git a/kokoro/release/python/linux/build_artifacts.sh b/kokoro/release/python/linux/build_artifacts.sh new file mode 100755 index 00000000..032e30d5 --- /dev/null +++ b/kokoro/release/python/linux/build_artifacts.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -ex + +# change to repo root +pushd $(dirname $0)/../../../.. + +export REPO_DIR=protobuf +export BUILD_VERSION=`grep -i "version" python/google/protobuf/__init__.py | grep -o "'.*'" | tr -d "'"` +export BUILD_COMMIT=v$BUILD_VERSION +export PLAT=x86_64 +export UNICODE_WIDTH=32 +export MACOSX_DEPLOYMENT_TARGET=10.9 + +mkdir artifacts +export ARTIFACT_DIR=$(pwd)/artifacts + +git clone https://github.com/matthew-brett/multibuild.git +cp kokoro/release/python/linux/config.sh config.sh + +build_artifact_version() { + MB_PYTHON_VERSION=$1 + + # Clean up env + rm -rf venv + sudo rm -rf protobuf + git clone https://github.com/google/protobuf.git + + source multibuild/common_utils.sh + source multibuild/travis_steps.sh + before_install + + clean_code $REPO_DIR $BUILD_COMMIT + sed -i '/Wno-sign-compare/a \ \ \ \ \ \ \ \ extra_compile_args.append("-std=c++11")' $REPO_DIR/python/setup.py + cat $REPO_DIR/python/setup.py + + build_wheel $REPO_DIR/python $PLAT + + mv wheelhouse/* $ARTIFACT_DIR +} + +build_artifact_version 2.7 +build_artifact_version 3.4 +build_artifact_version 3.5 +build_artifact_version 3.6 diff --git a/kokoro/release/python/linux/config.sh b/kokoro/release/python/linux/config.sh new file mode 100644 index 00000000..bfa8a009 --- /dev/null +++ b/kokoro/release/python/linux/config.sh @@ -0,0 +1,48 @@ +# Define custom utilities +# Test for OSX with [ -n "$IS_OSX" ] + +function pre_build { + # Any stuff that you need to do before you start building the wheels + # Runs in the root directory of this repository. + pushd protobuf + + # Build protoc + ./autogen.sh + ./configure + + CXXFLAGS="-fPIC -g -O2" ./configure + make -j8 + + # Generate python dependencies. + pushd python + python setup.py build_py + popd + + popd +} + +function bdist_wheel_cmd { + # Builds wheel with bdist_wheel, puts into wheelhouse + # + # It may sometimes be useful to use bdist_wheel for the wheel building + # process. For example, versioneer has problems with versions which are + # fixed with bdist_wheel: + # https://github.com/warner/python-versioneer/issues/121 + local abs_wheelhouse=$1 + + # Modify build version + pwd + ls + python setup.py bdist_wheel --cpp_implementation --compile_static_extension + cp dist/*.whl $abs_wheelhouse +} + +function build_wheel { + build_wheel_cmd "bdist_wheel_cmd" $@ +} + +function run_tests { + # Runs tests on installed distribution from an empty directory + python --version + python -c "from google.protobuf.pyext import _message;" +} diff --git a/kokoro/release/python/linux/release.cfg b/kokoro/release/python/linux/release.cfg new file mode 100644 index 00000000..b31b3f11 --- /dev/null +++ b/kokoro/release/python/linux/release.cfg @@ -0,0 +1,8 @@ +# Configuration for Linux release builds +build_file: "protobuf/kokoro/release/python/linux/build_artifacts.sh" + +action { + define_artifacts { + regex: "github/protobuf/artifacts/**" + } +} diff --git a/kokoro/release/python/macos/build_artifacts.sh b/kokoro/release/python/macos/build_artifacts.sh new file mode 100755 index 00000000..2aec5e64 --- /dev/null +++ b/kokoro/release/python/macos/build_artifacts.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +set -ex + +# change to repo root +pushd $(dirname $0)/../../../.. + +export REPO_DIR=protobuf +export BUILD_VERSION=`grep -i "version" python/google/protobuf/__init__.py | grep -o "'.*'" | tr -d "'"` +export BUILD_COMMIT=v$BUILD_VERSION +export PLAT=x86_64 +export UNICODE_WIDTH=32 +export MACOSX_DEPLOYMENT_TARGET=10.9 +export TRAVIS_OS_NAME="osx" + +mkdir artifacts +export ARTIFACT_DIR=$(pwd)/artifacts + +git clone https://github.com/matthew-brett/multibuild.git +cp kokoro/release/python/linux/config.sh config.sh + +OLD_PATH=$PATH + +build_artifact_version() { + MB_PYTHON_VERSION=$1 + + # Clean up env + rm -rf venv + sudo rm -rf protobuf + git clone https://github.com/google/protobuf.git + export PATH=$OLD_PATH + + source multibuild/common_utils.sh + source multibuild/travis_steps.sh + before_install + + clean_code $REPO_DIR $BUILD_COMMIT + + build_wheel $REPO_DIR/python $PLAT + + mv wheelhouse/* $ARTIFACT_DIR +} + +build_artifact_version 2.7 +build_artifact_version 3.4 +build_artifact_version 3.5 +build_artifact_version 3.6 diff --git a/kokoro/release/python/macos/release.cfg b/kokoro/release/python/macos/release.cfg new file mode 100644 index 00000000..85cf81b5 --- /dev/null +++ b/kokoro/release/python/macos/release.cfg @@ -0,0 +1,8 @@ +# Configuration for Mac OSX release builds +build_file: "protobuf/kokoro/release/python/macos/build_artifacts.sh" + +action { + define_artifacts { + regex: "github/protobuf/artifacts/**" + } +} diff --git a/kokoro/release/python/windows/build_artifacts.bat b/kokoro/release/python/windows/build_artifacts.bat new file mode 100644 index 00000000..86270e6f --- /dev/null +++ b/kokoro/release/python/windows/build_artifacts.bat @@ -0,0 +1,53 @@ +REM Move scripts to root +cd github\protobuf +copy kokoro\release\python\windows\build_single_artifact.bat build_single_artifact.bat + +REM Set environment variables +set REPO_DIR=protobuf +set PACKAGE_NAME=protobuf +set BUILD_DLL=OFF +set UNICODE=ON +set PB_TEST_DEP="six==1.9" +set OTHER_TEST_DEP="setuptools==38.5.1" +set OLD_PATH=C:\Program Files (x86)\MSBuild\14.0\bin\;%PATH% +for /f "tokens=*" %%i in ( 'grep -i "version" python/google/protobuf/__init__.py ^| grep -o "'.*'"' ) do set BUILD_VERSION=%%i +set BUILD_COMMIT=v%BUILD_VERSION:'=% + +REM Fetch multibuild +git clone https://github.com/matthew-brett/multibuild.git + +REM Install zlib +mkdir zlib +curl -L -o zlib.zip http://www.winimage.com/zLibDll/zlib123dll.zip +curl -L -o zlib-src.zip http://www.winimage.com/zLibDll/zlib123.zip +7z x zlib.zip -ozlib +7z x zlib-src.zip -ozlib\include +SET ZLIB_ROOT=%cd%\zlib +del /Q zlib.zip +del /Q zlib-src.zip + +REM Create directory for artifacts +SET ARTIFACT_DIR=%cd%\artifacts +mkdir %ARTIFACT_DIR% + +REM Build wheel + +SET PYTHON=C:\python35_32bit +SET PYTHON_VERSION=3.5 +SET PYTHON_ARCH=32 +CALL build_single_artifact.bat + +SET PYTHON=C:\python35 +SET PYTHON_VERSION=3.5 +SET PYTHON_ARCH=64 +CALL build_single_artifact.bat + +SET PYTHON=C:\python36_32bit +SET PYTHON_VERSION=3.6 +SET PYTHON_ARCH=32 +CALL build_single_artifact.bat + +SET PYTHON=C:\python36 +SET PYTHON_VERSION=3.6 +SET PYTHON_ARCH=64 +CALL build_single_artifact.bat diff --git a/kokoro/release/python/windows/build_single_artifact.bat b/kokoro/release/python/windows/build_single_artifact.bat new file mode 100644 index 00000000..9d581992 --- /dev/null +++ b/kokoro/release/python/windows/build_single_artifact.bat @@ -0,0 +1,62 @@ +setlocal + +if %PYTHON%==C:\python35_32bit set generator=Visual Studio 14 +if %PYTHON%==C:\python35_32bit set vcplatform=Win32 + +if %PYTHON%==C:\python35 set generator=Visual Studio 14 Win64 +if %PYTHON%==C:\python35 set vcplatform=x64 + +if %PYTHON%==C:\python36_32bit set generator=Visual Studio 14 +if %PYTHON%==C:\python36_32bit set vcplatform=Win32 + +if %PYTHON%==C:\python36 set generator=Visual Studio 14 Win64 +if %PYTHON%==C:\python36 set vcplatform=x64 + +REM Prepend newly installed Python to the PATH of this build (this cannot be +REM done from inside the powershell script as it would require to restart +REM the parent CMD process). +SET PATH=%PYTHON%;%PYTHON%\Scripts;%OLD_PATH% +python -m pip install -U pip +pip install wheel + +REM Check that we have the expected version and architecture for Python +python --version +python -c "import struct; print(struct.calcsize('P') * 8)" + +rmdir /s/q protobuf +git clone https://github.com/google/protobuf.git + +REM Checkout release commit +cd %REPO_DIR% +git checkout %BUILD_COMMIT% + +REM ====================== +REM Build Protobuf Library +REM ====================== + +mkdir src\.libs + +mkdir vcprojects +pushd vcprojects +cmake -G "%generator%" -Dprotobuf_BUILD_SHARED_LIBS=%BUILD_DLL% -Dprotobuf_UNICODE=%UNICODE% -Dprotobuf_BUILD_TESTS=OFF ../cmake +msbuild protobuf.sln /p:Platform=%vcplatform% /p:Configuration=Release +dir /s /b +popd +copy vcprojects\Release\libprotobuf.lib src\.libs\libprotobuf.a +copy vcprojects\Release\libprotobuf-lite.lib src\.libs\libprotobuf-lite.a +SET PATH=%cd%\vcprojects\Release;%PATH% +dir vcprojects\Release + +REM ====================== +REM Build python library +REM ====================== + +cd python + +REM sed -i 's/\ extra_compile_args\ =\ \[\]/\ extra_compile_args\ =\ \[\'\/MT\'\]/g' setup.py + +python setup.py bdist_wheel --cpp_implementation --compile_static_extension +dir dist +copy dist\* %ARTIFACT_DIR% +dir %ARTIFACT_DIR% +cd ..\.. diff --git a/kokoro/release/python/windows/release.cfg b/kokoro/release/python/windows/release.cfg new file mode 100644 index 00000000..52616064 --- /dev/null +++ b/kokoro/release/python/windows/release.cfg @@ -0,0 +1,8 @@ +# Configuration for Windows protoc release builds +build_file: "protobuf/kokoro/release/python/windows/build_artifacts.bat" + +action { + define_artifacts { + regex: "github/protobuf/artifacts/**" + } +} diff --git a/kokoro/release/ruby/linux/build_artifacts.sh b/kokoro/release/ruby/linux/build_artifacts.sh new file mode 100755 index 00000000..88555226 --- /dev/null +++ b/kokoro/release/ruby/linux/build_artifacts.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -ex + +# change to repo root +cd $(dirname $0)/../../../.. + +source kokoro/release/ruby/linux/prepare_build.sh + +# ruby environment +source kokoro/release/ruby/linux/ruby/ruby_build_environment.sh + +# build artifacts +bash kokoro/release/ruby/linux/ruby/ruby_build.sh diff --git a/kokoro/release/linux/prepare_build.sh b/kokoro/release/ruby/linux/prepare_build.sh index d3ff8764..d3ff8764 100755 --- a/kokoro/release/linux/prepare_build.sh +++ b/kokoro/release/ruby/linux/prepare_build.sh diff --git a/kokoro/release/linux/release.cfg b/kokoro/release/ruby/linux/release.cfg index 5dae7019..dbc71b2c 100644 --- a/kokoro/release/linux/release.cfg +++ b/kokoro/release/ruby/linux/release.cfg @@ -1,5 +1,5 @@ # Configuration for Linux release builds -build_file: "protobuf/kokoro/release/linux/build_artifacts.sh" +build_file: "protobuf/kokoro/release/ruby/linux/build_artifacts.sh" action { define_artifacts { diff --git a/kokoro/release/linux/ruby/ruby_build.sh b/kokoro/release/ruby/linux/ruby/ruby_build.sh index cf6b433d..cf6b433d 100755 --- a/kokoro/release/linux/ruby/ruby_build.sh +++ b/kokoro/release/ruby/linux/ruby/ruby_build.sh diff --git a/kokoro/release/linux/ruby/ruby_build_environment.sh b/kokoro/release/ruby/linux/ruby/ruby_build_environment.sh index ea04f905..ea04f905 100755 --- a/kokoro/release/linux/ruby/ruby_build_environment.sh +++ b/kokoro/release/ruby/linux/ruby/ruby_build_environment.sh diff --git a/kokoro/release/macos/build_artifacts.sh b/kokoro/release/ruby/macos/build_artifacts.sh index 23135ca3..c68b63cc 100755 --- a/kokoro/release/macos/build_artifacts.sh +++ b/kokoro/release/ruby/macos/build_artifacts.sh @@ -3,17 +3,17 @@ set -ex # change to repo root -cd $(dirname $0)/../../.. +cd $(dirname $0)/../../../.. # all artifacts come here mkdir artifacts export ARTIFACT_DIR=$(pwd)/artifacts # ruby environment -bash kokoro/release/macos/ruby/ruby_build_environment.sh +bash kokoro/release/ruby/macos/ruby/ruby_build_environment.sh gem install rubygems-update update_rubygems # build artifacts -bash kokoro/release/macos/ruby/ruby_build.sh +bash kokoro/release/ruby/macos/ruby/ruby_build.sh diff --git a/kokoro/release/macos/release.cfg b/kokoro/release/ruby/macos/release.cfg index 35d36dee..cb1c08bb 100644 --- a/kokoro/release/macos/release.cfg +++ b/kokoro/release/ruby/macos/release.cfg @@ -1,5 +1,5 @@ # Configuration for Mac OSX release builds -build_file: "protobuf/kokoro/release/macos/build_artifacts.sh" +build_file: "protobuf/kokoro/release/ruby/macos/build_artifacts.sh" action { define_artifacts { diff --git a/kokoro/release/macos/ruby/ruby_build.sh b/kokoro/release/ruby/macos/ruby/ruby_build.sh index cf6b433d..cf6b433d 100755 --- a/kokoro/release/macos/ruby/ruby_build.sh +++ b/kokoro/release/ruby/macos/ruby/ruby_build.sh diff --git a/kokoro/release/macos/ruby/ruby_build_environment.sh b/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh index e9c619cd..e9c619cd 100755 --- a/kokoro/release/macos/ruby/ruby_build_environment.sh +++ b/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh |