From c5a2a7c3dba6a90a52689277d5a4eb08a5a383d2 Mon Sep 17 00:00:00 2001 From: Kun Zhang Date: Mon, 6 Apr 2015 14:31:29 -0700 Subject: Process to deploy artifacts for multiple platforms into a single release. - Do not close the staging repository automatically - Added staging.repository property - Updated README with instructions for deployment - Fix building 32-bit Mac artifact --- protoc-artifacts/README.md | 57 +++++++++++++++++++++++++++++++++++++++- protoc-artifacts/build-protoc.sh | 15 ++++++++++- protoc-artifacts/pom.xml | 5 ++++ 3 files changed, 75 insertions(+), 2 deletions(-) diff --git a/protoc-artifacts/README.md b/protoc-artifacts/README.md index 2f3ed218..8768a05b 100644 --- a/protoc-artifacts/README.md +++ b/protoc-artifacts/README.md @@ -54,7 +54,62 @@ Before you can upload artifacts to Maven Central repository, make sure you have read [this page](http://central.sonatype.org/pages/apache-maven.html) on how to configure GPG and Sonatype account. -Use the following command to upload artifacts: +You need to perform the deployment for every platform that you want to +suppport. DO NOT close the staging repository until you have done the +deployment for all platforms. + +Remove any ``SNAPSHOT`` or ``pre`` suffix from the version string before +deploying. + +Use the following command to deploy artifacts for the host platform to a +staging repository. ``` $ mvn clean deploy -P release ``` +It creates a new staging repository. Go to +https://oss.sonatype.org/#stagingRepositories and find the repository, usually +in the name like ``comgoogle-123``. + +You will want to run this command on a different platform. Remember, in +subsequent deployments you will need to provide the repository name that you +have found in the first deployment so that all artifacts go to the same +repository: +``` +$ mvn clean deploy -P release -Dstaging.repository=comgoogle-123 +``` + +A 32-bit artifact can be deployed from a 64-bit host with +``-Dos.detected.arch=x86_32`` + +When you have done deployment for all platforms, go to +https://oss.sonatype.org/#stagingRepositories, verify that the staging +repository has all the binaries, close and release this repository. + +### Tips for deploying on Windows +Under Windows the following error may occur: ``gpg: cannot open tty `no tty': +No such file or directory``. This can be fixed by configuring gpg through an +active profile in ``.m2\settings.xml`` where also the Sonatype password is +stored: +```xml + + + + ossrh + [username] + [password] + + + + + gpg + + gpg + [password] + + + + + gpg + + +``` diff --git a/protoc-artifacts/build-protoc.sh b/protoc-artifacts/build-protoc.sh index 50c4b349..714b1bec 100755 --- a/protoc-artifacts/build-protoc.sh +++ b/protoc-artifacts/build-protoc.sh @@ -71,7 +71,13 @@ checkArch () fi elif [[ "$OS" == osx ]]; then format="$(file -b "$1" | grep -o "[^ ]*$")" - assertEq $format "x86_64" $LINENO + if [[ "$ARCH" == x86_32 ]]; then + assertEq $format "i386" $LINENO + elif [[ "$ARCH" == x86_64 ]]; then + assertEq $format "x86_64" $LINENO + else + fail "Unsupported arch: $ARCH" + fi else fail "Unsupported system: $(uname)" fi @@ -120,6 +126,13 @@ elif [[ "$(uname)" == Linux* ]]; then fi elif [[ "$(uname)" == Darwin* ]]; then assertEq "$OS" osx $LINENO + if [[ "$ARCH" == x86_64 ]]; then + CXXFLAGS="$CXXFLAGS -m64" + elif [[ "$ARCH" == x86_32 ]]; then + CXXFLAGS="$CXXFLAGS -m32" + else + fail "Unsupported arch: $ARCH" + fi else fail "Unsupported system: $(uname)" fi diff --git a/protoc-artifacts/pom.xml b/protoc-artifacts/pom.xml index 7db7b53d..6cfcf057 100644 --- a/protoc-artifacts/pom.xml +++ b/protoc-artifacts/pom.xml @@ -90,6 +90,9 @@ release + + + @@ -114,7 +117,9 @@ sonatype-nexus-staging https://oss.sonatype.org/ + true false + ${staging.repository} -- cgit v1.2.3 From c6792363667c7ade6561f9e2736696a5c5bf2612 Mon Sep 17 00:00:00 2001 From: Kun Zhang Date: Tue, 7 Apr 2015 17:18:46 -0700 Subject: List the platforms that we currently support. Document staging.repository. --- protoc-artifacts/README.md | 7 ++++++- protoc-artifacts/pom.xml | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/protoc-artifacts/README.md b/protoc-artifacts/README.md index 8768a05b..3a530197 100644 --- a/protoc-artifacts/README.md +++ b/protoc-artifacts/README.md @@ -56,7 +56,12 @@ configure GPG and Sonatype account. You need to perform the deployment for every platform that you want to suppport. DO NOT close the staging repository until you have done the -deployment for all platforms. +deployment for all platforms. Currently the following platforms are supported: +- Linux (x86_32 and x86_64) +- Windows (x86_32 and x86_64) with + - Cygwin with MinGW compilers (both x86_32 and x86_64) + - MSYS with MinGW32 (x86_32 only) +- MacOSX (x86_32 and x86_64) Remove any ``SNAPSHOT`` or ``pre`` suffix from the version string before deploying. diff --git a/protoc-artifacts/pom.xml b/protoc-artifacts/pom.xml index 6cfcf057..fad81b57 100644 --- a/protoc-artifacts/pom.xml +++ b/protoc-artifacts/pom.xml @@ -91,6 +91,11 @@ release + -- cgit v1.2.3 From 6f2bc19bb3401389b71b515a03fb7c144127efee Mon Sep 17 00:00:00 2001 From: Kun Zhang Date: Tue, 7 Apr 2015 20:43:20 -0700 Subject: Add cross-compilation for Windows on Linux using MinGW. Check library dependencies of artifact --- protoc-artifacts/build-protoc.sh | 82 ++++++++++++++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 12 deletions(-) diff --git a/protoc-artifacts/build-protoc.sh b/protoc-artifacts/build-protoc.sh index 714b1bec..686010c0 100755 --- a/protoc-artifacts/build-protoc.sh +++ b/protoc-artifacts/build-protoc.sh @@ -25,7 +25,8 @@ E_ASSERT_FAILED=99 # Usage: fail() { - echo "Error: $1" + echo "ERROR: $1" + echo exit $E_ASSERT_FAILED } @@ -49,8 +50,11 @@ assertEq () # Usage: checkArch checkArch () { + echo + echo "Checking for file format ..." if [[ "$OS" == windows || "$OS" == linux ]]; then format="$(objdump -f "$1" | grep -o "file format .*$" | grep -o "[^ ]*$")" + echo Format=$format if [[ "$OS" == linux ]]; then if [[ "$ARCH" == x86_32 ]]; then assertEq $format "elf32-i386" $LINENO @@ -71,6 +75,7 @@ checkArch () fi elif [[ "$OS" == osx ]]; then format="$(file -b "$1" | grep -o "[^ ]*$")" + echo Format=$format if [[ "$ARCH" == x86_32 ]]; then assertEq $format "i386" $LINENO elif [[ "$ARCH" == x86_64 ]]; then @@ -79,8 +84,45 @@ checkArch () fail "Unsupported arch: $ARCH" fi else - fail "Unsupported system: $(uname)" + fail "Unsupported system: $OS" fi + echo +} + +# Checks the dependencies of the artifact. Artifacts should only depend on +# system libraries. +# Usage: checkDependencies +checkDependencies () +{ + if [[ "$OS" == windows ]]; then + dump_cmd='objdump -x '"$1"' | fgrep "DLL Name"' + white_list="KERNEL32\.dll\|msvcrt\.dll" + elif [[ "$OS" == linux ]]; then + dump_cmd='ldd '"$1" + if [[ "$ARCH" == x86_32 ]]; then + white_list="linux-gate\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|ld-linux\.so\.2" + elif [[ "$ARCH" == x86_64 ]]; then + white_list="linux-vdso\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|ld-linux-x86-64\.so\.2" + fi + elif [[ "$OS" == osx ]]; then + dump_cmd='otool -L '"$1" + white_list="libz\.1\.dylib\|libc++\.1\.dylib\|libSystem\.B\.dylib" + fi + if [[ -z "$white_list" || -z "$dump_cmd" ]]; then + fail "Unsupported platform $OS-$ARCH." + fi + echo "Checking for expected dependencies ..." + eval $dump_cmd | grep -i "$white_list" || fail "doesn't show any expected dependencies" + echo "Checking for unexpected dependencies ..." + eval $dump_cmd | grep -i -v "$white_list" + ret=$? + if [[ $ret == 0 ]]; then + fail "found unexpected dependencies (listed above)." + elif [[ $ret != 1 ]]; then + fail "Error when checking dependencies." + fi # grep returns 1 when "not found", which is what we expect + echo "Dependencies look good." + echo } ############################################################################ @@ -100,6 +142,7 @@ fi # Override the default value set in configure.ac that has '-g' which produces # huge binary. CXXFLAGS="-DNDEBUG" +LDFLAGS="" if [[ "$(uname)" == CYGWIN* ]]; then assertEq "$OS" windows $LINENO @@ -116,13 +159,28 @@ elif [[ "$(uname)" == MINGW32* ]]; then assertEq "$OS" windows $LINENO assertEq "$ARCH" x86_32 $LINENO elif [[ "$(uname)" == Linux* ]]; then - assertEq "$OS" linux $LINENO - if [[ "$ARCH" == x86_64 ]]; then - CXXFLAGS="$CXXFLAGS -m64" - elif [[ "$ARCH" == x86_32 ]]; then - CXXFLAGS="$CXXFLAGS -m32" + if [[ "$OS" == linux ]]; then + if [[ "$ARCH" == x86_64 ]]; then + CXXFLAGS="$CXXFLAGS -m64" + elif [[ "$ARCH" == x86_32 ]]; then + CXXFLAGS="$CXXFLAGS -m32" + else + fail "Unsupported arch: $ARCH" + fi + elif [[ "$OS" == windows ]]; then + # Cross-compilation for Windows + # TODO(zhangkun83) MinGW 64 always adds dependency on libwinpthread-1.dll, + # which is undesirable for repository deployment. + CONFIGURE_ARGS="$CONFIGURE_ARGS" + if [[ "$ARCH" == x86_64 ]]; then + CONFIGURE_ARGS="$CONFIGURE_ARGS --host=x86_64-w64-mingw32" + elif [[ "$ARCH" == x86_32 ]]; then + CONFIGURE_ARGS="$CONFIGURE_ARGS --host=i686-w64-mingw32" + else + fail "Unsupported arch: $ARCH" + fi else - fail "Unsupported arch: $ARCH" + fail "Cannot build $OS on $(uname)" fi elif [[ "$(uname)" == Darwin* ]]; then assertEq "$OS" osx $LINENO @@ -137,19 +195,19 @@ else fail "Unsupported system: $(uname)" fi -export CXXFLAGS - # Statically link libgcc and libstdc++. # -s to produce stripped binary. # And they don't work under Mac. if [[ "$OS" != osx ]]; then - export LDFLAGS="-static-libgcc -static-libstdc++ -s" + LDFLAGS="$LDFLAGS -static-libgcc -static-libstdc++ -s" fi +export CXXFLAGS LDFLAGS + TARGET_FILE=target/protoc.exe cd "$WORKING_DIR"/.. && ./configure $CONFIGURE_ARGS && cd src && make clean && make $MAKE_TARGET && cd "$WORKING_DIR" && mkdir -p target && (cp ../src/protoc $TARGET_FILE || cp ../src/protoc.exe $TARGET_FILE) && - checkArch $TARGET_FILE + checkArch $TARGET_FILE && checkDependencies $TARGET_FILE -- cgit v1.2.3 From 5c265faedbd93d867871c8739ae46e8f7f347434 Mon Sep 17 00:00:00 2001 From: Kun Zhang Date: Tue, 7 Apr 2015 21:06:37 -0700 Subject: Fix osx dependency check. otool prints the file name in the first line. Should skip it --- protoc-artifacts/build-protoc.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/protoc-artifacts/build-protoc.sh b/protoc-artifacts/build-protoc.sh index 686010c0..4027f11e 100755 --- a/protoc-artifacts/build-protoc.sh +++ b/protoc-artifacts/build-protoc.sh @@ -51,7 +51,7 @@ assertEq () checkArch () { echo - echo "Checking for file format ..." + echo "Checking file format ..." if [[ "$OS" == windows || "$OS" == linux ]]; then format="$(objdump -f "$1" | grep -o "file format .*$" | grep -o "[^ ]*$")" echo Format=$format @@ -105,7 +105,7 @@ checkDependencies () white_list="linux-vdso\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|ld-linux-x86-64\.so\.2" fi elif [[ "$OS" == osx ]]; then - dump_cmd='otool -L '"$1" + dump_cmd='otool -L '"$1"' | fgrep dylib' white_list="libz\.1\.dylib\|libc++\.1\.dylib\|libSystem\.B\.dylib" fi if [[ -z "$white_list" || -z "$dump_cmd" ]]; then -- cgit v1.2.3 From 62903ec0753c59728145bc8959db3c2b028f8eee Mon Sep 17 00:00:00 2001 From: Kun Zhang Date: Wed, 8 Apr 2015 00:14:36 -0700 Subject: Make the osx binary compatible with 10.7 and later --- protoc-artifacts/build-protoc.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/protoc-artifacts/build-protoc.sh b/protoc-artifacts/build-protoc.sh index 4027f11e..10c13074 100755 --- a/protoc-artifacts/build-protoc.sh +++ b/protoc-artifacts/build-protoc.sh @@ -106,7 +106,7 @@ checkDependencies () fi elif [[ "$OS" == osx ]]; then dump_cmd='otool -L '"$1"' | fgrep dylib' - white_list="libz\.1\.dylib\|libc++\.1\.dylib\|libSystem\.B\.dylib" + white_list="libz\.1\.dylib\|libstdc++\.6\.dylib\|libSystem\.B\.dylib" fi if [[ -z "$white_list" || -z "$dump_cmd" ]]; then fail "Unsupported platform $OS-$ARCH." @@ -184,6 +184,8 @@ elif [[ "$(uname)" == Linux* ]]; then fi elif [[ "$(uname)" == Darwin* ]]; then assertEq "$OS" osx $LINENO + # Make the binary compatible with OSX 10.7 and later + CXXFLAGS="$CXXFLAGS -mmacosx-version-min=10.7" if [[ "$ARCH" == x86_64 ]]; then CXXFLAGS="$CXXFLAGS -m64" elif [[ "$ARCH" == x86_32 ]]; then -- cgit v1.2.3 From 1c12612bad4e44beae09c43ea1171183d21aea49 Mon Sep 17 00:00:00 2001 From: Kun Zhang Date: Wed, 8 Apr 2015 10:39:21 -0700 Subject: Strip Mac artifacts --- protoc-artifacts/build-protoc.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/protoc-artifacts/build-protoc.sh b/protoc-artifacts/build-protoc.sh index 10c13074..96ca97c2 100755 --- a/protoc-artifacts/build-protoc.sh +++ b/protoc-artifacts/build-protoc.sh @@ -211,5 +211,12 @@ TARGET_FILE=target/protoc.exe cd "$WORKING_DIR"/.. && ./configure $CONFIGURE_ARGS && cd src && make clean && make $MAKE_TARGET && cd "$WORKING_DIR" && mkdir -p target && - (cp ../src/protoc $TARGET_FILE || cp ../src/protoc.exe $TARGET_FILE) && - checkArch $TARGET_FILE && checkDependencies $TARGET_FILE + (cp ../src/protoc $TARGET_FILE || cp ../src/protoc.exe $TARGET_FILE) || + exit 1 + +if [[ "$OS" == osx ]]; then + # Since Mac linker doesn't accept "-s", we need to run strip + strip $TARGET_FILE || exit 1 +fi + +checkArch $TARGET_FILE && checkDependencies $TARGET_FILE -- cgit v1.2.3