Sync travis/jenkins config files with master.

... and keep only C++, Java and Python tests running.

Change-Id: I399a7e9d44477fc72951d768d5a809324f4c25ab

4 files changed, 64 insertions, 113 deletions

diff --git a/.travis.yml b/.travis.yml
index 7cfe0dfb..68718a01 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,87 +6,12 @@ sudo: required
 language: cpp
 os:
   - linux
-  - osx
-# The Objective C build needs Xcode 7.0 or later.
-osx_image: xcode7.3
 script:
   - ./tests.sh $CONFIG
 env:
   - CONFIG=cpp
   - CONFIG=cpp_distcheck
-  - CONFIG=golang
-  - CONFIG=java_jdk6
   - CONFIG=java_jdk7
-  - CONFIG=java_oracle7
-  - CONFIG=javanano_jdk6
-  - CONFIG=javanano_jdk7
-  - CONFIG=javanano_oracle7
-  - CONFIG=javascript
-  # iOS build log was starting to choke travis UI, so split to cover the
-  # Xcode Debug and Release Configurations independently.
-  - CONFIG=objectivec_ios_debug
-  - CONFIG=objectivec_ios_release
-  - CONFIG=objectivec_osx
-  - CONFIG=objectivec_cocoapods_integration
   - CONFIG=python
-  - CONFIG=python_cpp
-  - CONFIG=ruby19
-  - CONFIG=ruby20
-  - CONFIG=ruby21
-  - CONFIG=ruby22
-  - CONFIG=jruby
-matrix:
-  exclude:
-    # It's nontrivial to programmatically install a new JDK from the command
-    # line on OS X, so we rely on testing on Linux for Java code.
-    - os: osx
-      env: CONFIG=java_jdk6
-    - os: osx
-      env: CONFIG=java_jdk7
-    - os: osx
-      env: CONFIG=java_oracle7
-    - os: osx
-      env: CONFIG=javanano_jdk6
-    - os: osx
-      env: CONFIG=javanano_jdk7
-    - os: osx
-      env: CONFIG=javanano_oracle7
-    # Requires installing golang, currently travis.sh is doing that with apt-get
-    # which doesn't work on OS X.
-    - os: osx
-      env: CONFIG=golang
-    # OS X/iOS tests of Objective C (needs Xcode, so it won't work on other
-    # platforms).
-    - os: linux
-      env: CONFIG=objectivec_ios_debug
-    - os: linux
-      env: CONFIG=objectivec_ios_release
-    - os: linux
-      env: CONFIG=objectivec_osx
-    - os: linux
-      env: CONFIG=objectivec_cocoapods_integration
-  # The dotnet environment requires Ubuntu 14.04 or 16.04. This
-  # configuration is effectively an "extra" one, outside the
-  # autogenerated matrix.
-  include:
-    - os: linux
-      env: CONFIG=csharp
-      dist: trusty
-  allow_failures:
-    # These currently do not work on OS X but are being worked on by @haberman.
-    - os: osx
-      env: CONFIG=ruby22
-    - os: osx
-      env: CONFIG=jruby
-    # https://github.com/google/protobuf/issues/1253 - Started failing when
-    # we moved to an OS X image that is 10.11.
-    - os: osx
-      env: CONFIG=python_cpp
-    # Mark the iOS test as flakey as xcodebuild some times fails to start the
-    # iOS Simulator.
-    - os: osx
-      env: CONFIG=objectivec_ios_debug
-    - os: osx
-      env: CONFIG=objectivec_ios_release
 notifications:
   email: false
diff --git a/jenkins/docker/Dockerfile b/jenkins/docker/Dockerfile
index c928ac71..53ac38f3 100644
--- a/jenkins/docker/Dockerfile
+++ b/jenkins/docker/Dockerfile
@@ -87,7 +87,6 @@ RUN wget www.nuget.org/NuGet.exe -O /usr/local/bin/nuget.exe
 RUN pip install pip --upgrade
 RUN pip install virtualenv tox yattag
 
-
 ##################
 # Ruby dependencies
 
@@ -95,12 +94,12 @@ RUN pip install virtualenv tox yattag
 RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
 RUN \curl -sSL https://get.rvm.io | bash -s stable
 
-# Install Ruby 2.1
+# Install Ruby 2.1, Ruby 2.2 and JRuby 1.7
 RUN /bin/bash -l -c "rvm install ruby-2.1"
-RUN /bin/bash -l -c "rvm use --default ruby-2.1"
+RUN /bin/bash -l -c "rvm install ruby-2.2"
+RUN /bin/bash -l -c "rvm install jruby-1.7"
 RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc"
 RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc"
-RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc"
 RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc"
 
 ##################
@@ -118,12 +117,29 @@ RUN cd /tmp && \
   ./configure && \
   make -j6 && \
   cd java && \
-  $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO -P lite && \
   $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO && \
   cd ../javanano && \
   $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO
 
 ##################
+# Go dependencies.
+RUN apt-get install -y  \
+  # -- For go -- \
+  golang
+
+##################
+# Javascript dependencies.
+Run apt-get install -y \
+  # -- For javascript -- \
+  npm
+
+# On Debian/Ubuntu, nodejs binary is named 'nodejs' because the name 'node'
+# is taken by another legacy binary. We don't have that legacy binary and
+# npm expects the binary to be named 'node', so we just create a symbol
+# link here.
+RUN ln -s `which nodejs` /usr/bin/node
+
+##################
 # Prepare ccache
 
 RUN ln -s /usr/bin/ccache /usr/local/bin/gcc
diff --git a/jenkins/pull_request_in_docker.sh b/jenkins/pull_request_in_docker.sh
index 887f97c5..543f1bce 100755
--- a/jenkins/pull_request_in_docker.sh
+++ b/jenkins/pull_request_in_docker.sh
@@ -48,14 +48,10 @@ $TIME_CMD $TEST_SCRIPT cpp > >(tee $CPP_STDOUT) 2> >(tee $CPP_STDERR >&2)
 # Other tests are run in parallel.
 
 parallel --results $LOG_OUTPUT_DIR --joblog $OUTPUT_DIR/joblog $TEST_SCRIPT ::: \
-  csharp \
   java_jdk7 \
-  javanano_jdk7 \
   java_oracle7 \
-  javanano_oracle7 \
   python \
   python_cpp \
-  ruby21 \
   || true  # Process test results even if tests fail.
 
 cat $OUTPUT_DIR/joblog
diff --git a/tests.sh b/tests.sh
index ed901419..7d2eb3be 100755
--- a/tests.sh
+++ b/tests.sh
@@ -44,6 +44,30 @@ build_cpp() {
 build_cpp_distcheck() {
   ./autogen.sh
   ./configure
+  make dist
+
+  # List all files that should be included in the distribution package.
+  git ls-files | grep "^\(java\|python\|objectivec\|csharp\|js\|ruby\|cmake\|examples\)" |\
+      grep -v ".gitignore" | grep -v "java/compatibility_tests" > dist.lst
+  # Unzip the dist tar file.
+  DIST=`ls *.tar.gz`
+  tar -xf $DIST
+  cd ${DIST//.tar.gz}
+  # Check if every file exists in the dist tar file.
+  FILES_MISSING=""
+  for FILE in $(<../dist.lst); do
+    if ! file $FILE &>/dev/null; then
+      echo "$FILE is not found!"
+      FILES_MISSING="$FILE $FILES_MISSING"
+    fi
+  done
+  cd ..
+  if [ ! -z "$FILES_MISSING" ]; then
+    echo "Missing files in EXTRA_DIST: $FILES_MISSING"
+    exit 1
+  fi
+
+  # Do the regular dist-check for C++.
   make distcheck -j2
 }
 
@@ -63,7 +87,7 @@ build_csharp() {
     sudo apt-get install -qq mono-devel referenceassemblies-pcl nunit
     
     # Then install the dotnet SDK as per Ubuntu 14.04 instructions on dot.net.
-    sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet/ trusty main" > /etc/apt/sources.list.d/dotnetdev.list'
+    sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ trusty main" > /etc/apt/sources.list.d/dotnetdev.list'
     sudo apt-key adv --keyserver apt-mo.trafficmanager.net --recv-keys 417A0893
     sudo apt-get update -qq
     sudo apt-get install -qq dotnet-dev-1.0.0-preview2-003121
@@ -89,10 +113,12 @@ build_golang() {
   export PATH="`pwd`/src:$PATH"
 
   # Install Go and the Go protobuf compiler plugin.
-  sudo apt-get update -qq
-  sudo apt-get install -qq golang
+  on_travis sudo apt-get update -qq
+  on_travis sudo apt-get install -qq golang
+
   export GOPATH="$HOME/gocode"
   mkdir -p "$GOPATH/src/github.com/google"
+  rm -f "$GOPATH/src/github.com/google/protobuf"
   ln -s "`pwd`" "$GOPATH/src/github.com/google/protobuf"
   export PATH="$GOPATH/bin:$PATH"
   go get github.com/golang/protobuf/protoc-gen-go
@@ -103,13 +129,10 @@ build_golang() {
 use_java() {
   version=$1
   case "$version" in
-    jdk6)
-      on_travis sudo apt-get install openjdk-6-jdk
-      export PATH=/usr/lib/jvm/java-6-openjdk-amd64/bin:$PATH
-      ;;
     jdk7)
       on_travis sudo apt-get install openjdk-7-jdk
       export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH
+      export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
       ;;
     oracle7)
       if [ "$TRAVIS" == "true" ]; then
@@ -120,6 +143,7 @@ use_java() {
         yes | sudo apt-get install oracle-java7-installer
       fi;
       export PATH=/usr/lib/jvm/java-7-oracle/bin:$PATH
+      export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
       ;;
   esac
 
@@ -130,6 +154,7 @@ use_java() {
 
   which java
   java -version
+  $MVN -version
 }
 
 # --batch-mode supresses download progress output that spams the logs.
@@ -162,10 +187,6 @@ build_javanano() {
   cd javanano && $MVN test && cd ..
 }
 
-build_java_jdk6() {
-  use_java jdk6
-  build_java jdk6
-}
 build_java_jdk7() {
   use_java jdk7
   build_java_with_conformance_tests
@@ -175,10 +196,6 @@ build_java_oracle7() {
   build_java oracle7
 }
 
-build_javanano_jdk6() {
-  use_java jdk6
-  build_javanano
-}
 build_javanano_jdk7() {
   use_java jdk7
   build_javanano
@@ -281,14 +298,6 @@ build_python_cpp() {
   cd ..
 }
 
-build_ruby19() {
-  internal_build_cpp  # For conformance tests.
-  cd ruby && bash travis-test.sh ruby-1.9 && cd ..
-}
-build_ruby20() {
-  internal_build_cpp  # For conformance tests.
-  cd ruby && bash travis-test.sh ruby-2.0 && cd ..
-}
 build_ruby21() {
   internal_build_cpp  # For conformance tests.
   cd ruby && bash travis-test.sh ruby-2.1 && cd ..
@@ -299,7 +308,14 @@ build_ruby22() {
 }
 build_jruby() {
   internal_build_cpp  # For conformance tests.
-  cd ruby && bash travis-test.sh jruby && cd ..
+  # TODO(xiaofeng): Upgrade to jruby-9.x. There are some broken jests to be
+  # fixed.
+  cd ruby && bash travis-test.sh jruby-1.7 && cd ..
+}
+build_ruby_all() {
+  build_ruby21
+  build_ruby22
+  build_jruby
 }
 
 build_javascript() {
@@ -320,11 +336,10 @@ build_javascript() {
 if [ "$#" -ne 1 ]; then
   echo "
 Usage: $0 { cpp |
+            cpp_distcheck |
             csharp |
-            java_jdk6 |
             java_jdk7 |
             java_oracle7 |
-            javanano_jdk6 |
             javanano_jdk7 |
             javanano_oracle7 |
             objectivec_ios |
@@ -334,11 +349,10 @@ Usage: $0 { cpp |
             objectivec_cocoapods_integration |
             python |
             python_cpp |
-            ruby19 |
-            ruby20 |
             ruby21 |
             ruby22 |
-            jruby }
+            jruby |
+            ruby_all)
 "
   exit 1
 fi