diff options
author | Dan O'Reilly <oreilldf@gmail.com> | 2015-08-12 23:57:46 -0400 |
---|---|---|
committer | Dan O'Reilly <oreilldf@gmail.com> | 2015-08-12 23:57:46 -0400 |
commit | e47cdd5a559f488ba52756927ce68f4cf93874fa (patch) | |
tree | 8ce2723e822808baf58e96f569c86035717ea351 /protoc-artifacts | |
parent | daeaa6a28b81195f24d89222e649d79c9555af8b (diff) | |
parent | 38a56ee4b19d72c2e9d81a08b018704d1addf561 (diff) | |
download | protobuf-e47cdd5a559f488ba52756927ce68f4cf93874fa.tar.gz protobuf-e47cdd5a559f488ba52756927ce68f4cf93874fa.tar.bz2 protobuf-e47cdd5a559f488ba52756927ce68f4cf93874fa.zip |
Merge remote-tracking branch 'upstream/master' into py2_py3_straddle
Conflicts:
python/google/protobuf/descriptor_pool.py
python/google/protobuf/internal/api_implementation_default_test.py
python/google/protobuf/internal/cpp_message.py
python/google/protobuf/internal/descriptor_database_test.py
python/google/protobuf/internal/descriptor_pool_test.py
python/google/protobuf/internal/descriptor_python_test.py
python/google/protobuf/internal/descriptor_test.py
python/google/protobuf/internal/generator_test.py
python/google/protobuf/internal/message_factory_python_test.py
python/google/protobuf/internal/message_factory_test.py
python/google/protobuf/internal/message_test.py
python/google/protobuf/internal/proto_builder_test.py
python/google/protobuf/internal/python_message.py
python/google/protobuf/internal/reflection_test.py
python/google/protobuf/internal/service_reflection_test.py
python/google/protobuf/internal/symbol_database_test.py
python/google/protobuf/internal/text_encoding_test.py
python/google/protobuf/internal/text_format_test.py
python/google/protobuf/internal/unknown_fields_test.py
python/google/protobuf/internal/wire_format_test.py
python/google/protobuf/pyext/descriptor_cpp2_test.py
python/google/protobuf/pyext/message_factory_cpp2_test.py
python/google/protobuf/pyext/reflection_cpp2_generated_test.py
python/setup.py
ruby/lib/google/protobuf/message_exts.rb
Diffstat (limited to 'protoc-artifacts')
-rw-r--r-- | protoc-artifacts/Dockerfile | 40 | ||||
-rw-r--r-- | protoc-artifacts/README.md | 143 | ||||
-rwxr-xr-x | protoc-artifacts/build-protoc.sh | 225 | ||||
-rw-r--r-- | protoc-artifacts/pom.xml | 134 |
4 files changed, 542 insertions, 0 deletions
diff --git a/protoc-artifacts/Dockerfile b/protoc-artifacts/Dockerfile new file mode 100644 index 00000000..fd35b89f --- /dev/null +++ b/protoc-artifacts/Dockerfile @@ -0,0 +1,40 @@ +FROM centos:6.6 + +RUN yum install -y git \ + tar \ + wget \ + make \ + autoconf \ + curl-devel \ + unzip \ + automake \ + libtool \ + glibc-static.i686 \ + glibc-devel \ + glibc-devel.i686 + +# Install Java 8 +RUN wget -q --no-cookies --no-check-certificate \ + --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/8u45-b14/jdk-8u45-linux-x64.tar.gz" \ + -O - | tar xz -C /var/local +ENV JAVA_HOME /var/local/jdk1.8.0_45 +ENV PATH $JAVA_HOME/bin:$PATH + +# Install Maven +RUN wget -q http://apache.cs.utah.edu/maven/maven-3/3.3.3/binaries/apache-maven-3.3.3-bin.tar.gz -O - | \ + tar xz -C /var/local +ENV PATH /var/local/apache-maven-3.3.3/bin:$PATH + +# Install GCC 4.7 to support -static-libstdc++ +RUN wget http://people.centos.org/tru/devtools-1.1/devtools-1.1.repo -P /etc/yum.repos.d +RUN bash -c 'echo "enabled=1" >> /etc/yum.repos.d/devtools-1.1.repo' +RUN bash -c "sed -e 's/\$basearch/i386/g' /etc/yum.repos.d/devtools-1.1.repo > /etc/yum.repos.d/devtools-i386-1.1.repo" +RUN sed -e 's/testing-/testing-i386-/g' -i /etc/yum.repos.d/devtools-i386-1.1.repo +RUN yum install -y devtoolset-1.1 \ + devtoolset-1.1-libstdc++-devel \ + devtoolset-1.1-libstdc++-devel.i686 + +RUN git clone --depth 1 https://github.com/google/protobuf.git + +# Start in devtoolset environment that uses GCC 4.7 +CMD ["scl", "enable", "devtoolset-1.1", "bash"] diff --git a/protoc-artifacts/README.md b/protoc-artifacts/README.md new file mode 100644 index 00000000..06091cfe --- /dev/null +++ b/protoc-artifacts/README.md @@ -0,0 +1,143 @@ +# Build scripts that publish pre-compiled protoc artifacts +``protoc`` is the compiler for ``.proto`` files. It generates language bindings +for the messages and/or RPC services from ``.proto`` files. + +Because ``protoc`` is a native executable, the scripts under this directory +build and publish a ``protoc`` executable (a.k.a. artifact) to Maven +repositories. The artifact can be used by build automation tools so that users +would not need to compile and install ``protoc`` for their systems. + +## Versioning +The version of the ``protoc`` artifact must be the same as the version of the +Protobuf project. + +## Artifact name +The name of a published ``protoc`` artifact is in the following format: +``protoc-<version>-<os>-<arch>.exe``, e.g., ``protoc-3.0.0-alpha-3-windows-x86_64.exe``. + +## System requirement +Install [Apache Maven](http://maven.apache.org/) if you don't have it. + +The scripts only work under Unix-like environments, e.g., Linux, MacOSX, and +Cygwin or MinGW for Windows. Please see ``README.md`` of the Protobuf project +for how to set up the build environment. + +## To install artifacts locally +The following command will install the ``protoc`` artifact to your local Maven repository. +``` +$ mvn install +``` + +## Cross-compilation +The Maven script will try to detect the OS and the architecture from Java +system properties. It's possible to build a protoc binary for an architecture +that is different from what Java has detected, as long as you have the proper +compilers installed. + +You can override the Maven properties ``os.detected.name`` and +``os.detected.arch`` to force the script to generate binaries for a specific OS +and/or architecture. Valid values are defined as the return values of +``normalizeOs()`` and ``normalizeArch()`` of ``Detector`` from +[os-maven-plugin](https://github.com/trustin/os-maven-plugin/blob/master/src/main/java/kr/motd/maven/os/Detector.java). +Frequently used values are: +- ``os.detected.name``: ``linux``, ``osx``, ``windows``. +- ``os.detected.arch``: ``x86_32``, ``x86_64`` + +For example, MingGW32 only ships with 32-bit compilers, but you can still build +32-bit protoc under 64-bit Windows, with the following command: +``` +$ mvn install -Dos.detected.arch=x86_32 +``` + +## To push artifacts to Maven Central +Before you can upload artifacts to Maven Central repository, make sure you have +read [this page](http://central.sonatype.org/pages/apache-maven.html) on how to +configure GPG and Sonatype account. + +You need to perform the deployment for every platform that you want to +suppport. DO NOT close the staging repository until you have done the +deployment for all platforms. Currently the following platforms are supported: +- Linux (x86_32 and x86_64) +- Windows (x86_32 and x86_64) with + - Cygwin with MinGW compilers (both x86_32 and x86_64) + - MSYS with MinGW32 (x86_32 only) +- MacOSX (x86_32 and x86_64) + +Use the following command to deploy artifacts for the host platform to a +staging repository. +``` +$ mvn clean deploy -P release +``` +It creates a new staging repository. Go to +https://oss.sonatype.org/#stagingRepositories and find the repository, usually +in the name like ``comgoogle-123``. + +You will want to run this command on a different platform. Remember, in +subsequent deployments you will need to provide the repository name that you +have found in the first deployment so that all artifacts go to the same +repository: +``` +$ mvn clean deploy -P release -Dstaging.repository=comgoogle-123 +``` + +A 32-bit artifact can be deployed from a 64-bit host with +``-Dos.detected.arch=x86_32`` + +When you have done deployment for all platforms, go to +https://oss.sonatype.org/#stagingRepositories, verify that the staging +repository has all the binaries, close and release this repository. + +### Tips for deploying on Linux +We build on Centos 6.6 to provide a good compatibility for not very new +systems. We have provided a ``Dockerfile`` under this directory to build the +environment. It has been tested with Docker 1.6.1. + +To build a image: +``` +$ docker build -t protoc-artifacts . +``` + +To run the image: +``` +$ docker run -it --rm=true protoc-artifacts +``` + +The Protobuf repository has been cloned into ``/protobuf``. + +### Tips for deploying on Windows +Under Windows the following error may occur: ``gpg: cannot open tty `no tty': +No such file or directory``. This can be fixed by configuring gpg through an +active profile in ``.m2\settings.xml`` where also the Sonatype password is +stored: +```xml +<settings> + <servers> + <server> + <id>ossrh</id> + <username>[username]</username> + <password>[password]</password> + </server> + </servers> + <profiles> + <profile> + <id>gpg</id> + <properties> + <gpg.executable>gpg</gpg.executable> + <gpg.passphrase>[password]</gpg.passphrase> + </properties> + </profile> + </profiles> + <activeProfiles> + <activeProfile>gpg</activeProfile> + </activeProfiles> +</settings> +``` + +### Tested build environments +We have succesfully built artifacts on the following environments: +- Linux x86_32 and x86_64: + - Centos 6.6 (within Docker 1.6.1) + - Ubuntu 14.04.2 64-bit +- Windows x86_32: MSYS with ``mingw32-gcc-g++ 4.8.1-4`` on Windows 7 64-bit +- Windows x86_64: Cygwin64 with ``mingw64-x86_64-gcc-g++ 4.8.3-1`` on Windows 7 64-bit +- Mac OS X x86_32 and x86_64: Mac OS X 10.9.5 diff --git a/protoc-artifacts/build-protoc.sh b/protoc-artifacts/build-protoc.sh new file mode 100755 index 00000000..2f67c508 --- /dev/null +++ b/protoc-artifacts/build-protoc.sh @@ -0,0 +1,225 @@ +#!/bin/bash + +# Builds protoc executable into target/protoc.exe +# To be run from Maven. +# Usage: build-protoc.sh <OS> <ARCH> +# <OS> and <ARCH> are ${os.detected.name} and ${os.detected.arch} from os-maven-plugin +OS=$1 +ARCH=$2 + +if [[ $# < 2 ]]; then + echo "No arguments provided. This script is intended to be run from Maven." + exit 1 +fi + +# Under Cygwin, bash doesn't have these in PATH when called from Maven which +# runs in Windows version of Java. +export PATH="/bin:/usr/bin:$PATH" + +############################################################################ +# Helper functions +############################################################################ +E_PARAM_ERR=98 +E_ASSERT_FAILED=99 + +# Usage: +fail() +{ + echo "ERROR: $1" + echo + exit $E_ASSERT_FAILED +} + +# Usage: assertEq VAL1 VAL2 $LINENO +assertEq () +{ + lineno=$3 + if [ -z "$lineno" ]; then + echo "lineno not given" + exit $E_PARAM_ERR + fi + + if [[ "$1" != "$2" ]]; then + echo "Assertion failed: \"$1\" == \"$2\"" + echo "File \"$0\", line $lineno" # Give name of file and line number. + exit $E_ASSERT_FAILED + fi +} + +# Checks the artifact is for the expected architecture +# Usage: checkArch <path-to-protoc> +checkArch () +{ + echo + echo "Checking file format ..." + if [[ "$OS" == windows || "$OS" == linux ]]; then + format="$(objdump -f "$1" | grep -o "file format .*$" | grep -o "[^ ]*$")" + echo Format=$format + if [[ "$OS" == linux ]]; then + if [[ "$ARCH" == x86_32 ]]; then + assertEq $format "elf32-i386" $LINENO + elif [[ "$ARCH" == x86_64 ]]; then + assertEq $format "elf64-x86-64" $LINENO + else + fail "Unsupported arch: $ARCH" + fi + else + # $OS == windows + if [[ "$ARCH" == x86_32 ]]; then + assertEq $format "pei-i386" $LINENO + elif [[ "$ARCH" == x86_64 ]]; then + assertEq $format "pei-x86-64" $LINENO + else + fail "Unsupported arch: $ARCH" + fi + fi + elif [[ "$OS" == osx ]]; then + format="$(file -b "$1" | grep -o "[^ ]*$")" + echo Format=$format + if [[ "$ARCH" == x86_32 ]]; then + assertEq $format "i386" $LINENO + elif [[ "$ARCH" == x86_64 ]]; then + assertEq $format "x86_64" $LINENO + else + fail "Unsupported arch: $ARCH" + fi + else + fail "Unsupported system: $OS" + fi + echo +} + +# Checks the dependencies of the artifact. Artifacts should only depend on +# system libraries. +# Usage: checkDependencies <path-to-protoc> +checkDependencies () +{ + if [[ "$OS" == windows ]]; then + dump_cmd='objdump -x '"$1"' | fgrep "DLL Name"' + white_list="KERNEL32\.dll\|msvcrt\.dll" + elif [[ "$OS" == linux ]]; then + dump_cmd='ldd '"$1" + if [[ "$ARCH" == x86_32 ]]; then + white_list="linux-gate\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|ld-linux\.so\.2" + elif [[ "$ARCH" == x86_64 ]]; then + white_list="linux-vdso\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|ld-linux-x86-64\.so\.2" + fi + elif [[ "$OS" == osx ]]; then + dump_cmd='otool -L '"$1"' | fgrep dylib' + white_list="libz\.1\.dylib\|libstdc++\.6\.dylib\|libSystem\.B\.dylib" + fi + if [[ -z "$white_list" || -z "$dump_cmd" ]]; then + fail "Unsupported platform $OS-$ARCH." + fi + echo "Checking for expected dependencies ..." + eval $dump_cmd | grep -i "$white_list" || fail "doesn't show any expected dependencies" + echo "Checking for unexpected dependencies ..." + eval $dump_cmd | grep -i -v "$white_list" + ret=$? + if [[ $ret == 0 ]]; then + fail "found unexpected dependencies (listed above)." + elif [[ $ret != 1 ]]; then + fail "Error when checking dependencies." + fi # grep returns 1 when "not found", which is what we expect + echo "Dependencies look good." + echo +} +############################################################################ + +echo "Building protoc, OS=$OS ARCH=$ARCH" + +# Nested double quotes are unintuitive, but it works. +cd "$(dirname "$0")" + +WORKING_DIR=$(pwd) +CONFIGURE_ARGS="--disable-shared" + +MAKE_TARGET="protoc" +if [[ "$OS" == windows ]]; then + MAKE_TARGET="${MAKE_TARGET}.exe" +fi + +# Override the default value set in configure.ac that has '-g' which produces +# huge binary. +CXXFLAGS="-DNDEBUG" +LDFLAGS="" + +if [[ "$(uname)" == CYGWIN* ]]; then + assertEq "$OS" windows $LINENO + # Use mingw32 compilers because executables produced by Cygwin compiler + # always have dependency on Cygwin DLL. + if [[ "$ARCH" == x86_64 ]]; then + CONFIGURE_ARGS="$CONFIGURE_ARGS --host=x86_64-w64-mingw32" + elif [[ "$ARCH" == x86_32 ]]; then + CONFIGURE_ARGS="$CONFIGURE_ARGS --host=i686-pc-mingw32" + else + fail "Unsupported arch by CYGWIN: $ARCH" + fi +elif [[ "$(uname)" == MINGW32* ]]; then + assertEq "$OS" windows $LINENO + assertEq "$ARCH" x86_32 $LINENO +elif [[ "$(uname)" == MINGW64* ]]; then + assertEq "$OS" windows $LINENO + assertEq "$ARCH" x86_64 $LINENO +elif [[ "$(uname)" == Linux* ]]; then + if [[ "$OS" == linux ]]; then + if [[ "$ARCH" == x86_64 ]]; then + CXXFLAGS="$CXXFLAGS -m64" + elif [[ "$ARCH" == x86_32 ]]; then + CXXFLAGS="$CXXFLAGS -m32" + else + fail "Unsupported arch: $ARCH" + fi + elif [[ "$OS" == windows ]]; then + # Cross-compilation for Windows + # TODO(zhangkun83) MinGW 64 always adds dependency on libwinpthread-1.dll, + # which is undesirable for repository deployment. + CONFIGURE_ARGS="$CONFIGURE_ARGS" + if [[ "$ARCH" == x86_64 ]]; then + CONFIGURE_ARGS="$CONFIGURE_ARGS --host=x86_64-w64-mingw32" + elif [[ "$ARCH" == x86_32 ]]; then + CONFIGURE_ARGS="$CONFIGURE_ARGS --host=i686-w64-mingw32" + else + fail "Unsupported arch: $ARCH" + fi + else + fail "Cannot build $OS on $(uname)" + fi +elif [[ "$(uname)" == Darwin* ]]; then + assertEq "$OS" osx $LINENO + # Make the binary compatible with OSX 10.7 and later + CXXFLAGS="$CXXFLAGS -mmacosx-version-min=10.7" + if [[ "$ARCH" == x86_64 ]]; then + CXXFLAGS="$CXXFLAGS -m64" + elif [[ "$ARCH" == x86_32 ]]; then + CXXFLAGS="$CXXFLAGS -m32" + else + fail "Unsupported arch: $ARCH" + fi +else + fail "Unsupported system: $(uname)" +fi + +# Statically link libgcc and libstdc++. +# -s to produce stripped binary. +# And they don't work under Mac. +if [[ "$OS" != osx ]]; then + LDFLAGS="$LDFLAGS -static-libgcc -static-libstdc++ -s" +fi + +export CXXFLAGS LDFLAGS + +TARGET_FILE=target/protoc.exe + +cd "$WORKING_DIR"/.. && ./configure $CONFIGURE_ARGS && + cd src && make clean && make google/protobuf/stubs/pbconfig.h $MAKE_TARGET && + cd "$WORKING_DIR" && mkdir -p target && + (cp ../src/protoc $TARGET_FILE || cp ../src/protoc.exe $TARGET_FILE) || + exit 1 + +if [[ "$OS" == osx ]]; then + # Since Mac linker doesn't accept "-s", we need to run strip + strip $TARGET_FILE || exit 1 +fi + +checkArch $TARGET_FILE && checkDependencies $TARGET_FILE diff --git a/protoc-artifacts/pom.xml b/protoc-artifacts/pom.xml new file mode 100644 index 00000000..52a7d2d6 --- /dev/null +++ b/protoc-artifacts/pom.xml @@ -0,0 +1,134 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.google</groupId> + <artifactId>google</artifactId> + <version>1</version> + </parent> + <groupId>com.google.protobuf</groupId> + <artifactId>protoc</artifactId> + <version>3.0.0-alpha-4-pre</version> + <packaging>pom</packaging> + <name>Protobuf Compiler</name> + <description> + Protobuf Compiler (protoc) is a compiler for .proto files. It generates + language-specific code for Protobuf messages and RPC interfaces. + </description> + <inceptionYear>2008</inceptionYear> + <url>https://developers.google.com/protocol-buffers/</url> + <licenses> + <license> + <name>New BSD license</name> + <url>http://www.opensource.org/licenses/bsd-license.php</url> + <distribution>repo</distribution> + </license> + </licenses> + <scm> + <url>https://github.com/google/protobuf</url> + <connection> + scm:git:https://github.com/google/protobuf.git + </connection> + </scm> + <build> + <extensions> + <extension> + <groupId>kr.motd.maven</groupId> + <artifactId>os-maven-plugin</artifactId> + <version>1.2.3.Final</version> + </extension> + </extensions> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>1.1.1</version> + <executions> + <execution> + <phase>compile</phase> + <goals> + <goal>exec</goal> + </goals> + </execution> + </executions> + <configuration> + <executable>bash</executable> + <arguments> + <argument>build-protoc.sh</argument> + <argument>${os.detected.name}</argument> + <argument>${os.detected.arch}</argument> + </arguments> + </configuration> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>1.8</version> + <executions> + <execution> + <id>attach-artifacts</id> + <phase>package</phase> + <goals> + <goal>attach-artifact</goal> + </goals> + <configuration> + <artifacts> + <artifact> + <file>${basedir}/target/protoc.exe</file> + <classifier>${os.detected.name}-${os.detected.arch}</classifier> + <type>exe</type> + </artifact> + </artifacts> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + <profiles> + <profile> + <id>release</id> + <properties> + <!-- Specify the staging repository to deploy to. This can be left + empty for the first deployment, and Sonatype will create one. For + subsequent deployments it should be set to what Sonatype has + created, so that all deployments will go to the same repository. + --> + <staging.repository></staging.repository> + </properties> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-gpg-plugin</artifactId> + <version>1.5</version> + <executions> + <execution> + <id>sign-artifacts</id> + <phase>verify</phase> + <goals> + <goal>sign</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.sonatype.plugins</groupId> + <artifactId>nexus-staging-maven-plugin</artifactId> + <version>1.6.3</version> + <extensions>true</extensions> + <configuration> + <serverId>sonatype-nexus-staging</serverId> + <nexusUrl>https://oss.sonatype.org/</nexusUrl> + <skipStagingRepositoryClose>true</skipStagingRepositoryClose> + <autoReleaseAfterClose>false</autoReleaseAfterClose> + <stagingRepositoryId>${staging.repository}</stagingRepositoryId> + </configuration> + </plugin> + </plugins> + </build> + </profile> + </profiles> +</project> |