diff options
-rw-r--r-- | .travis.yml | 2 | ||||
-rw-r--r-- | csharp/protos/unittest_issues.proto | 9 | ||||
-rw-r--r-- | csharp/src/Google.Protobuf.Test/IssuesTest.cs | 19 | ||||
-rw-r--r-- | csharp/src/Google.Protobuf.Test/TestProtos/UnittestIssues.cs | 173 | ||||
-rw-r--r-- | csharp/src/Google.Protobuf/JsonFormatter.cs | 6 | ||||
-rw-r--r-- | csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs | 6 | ||||
-rw-r--r-- | csharp/src/Google.Protobuf/Reflection/MessageDescriptor.cs | 2 | ||||
-rw-r--r-- | jenkins/README.md | 6 | ||||
-rwxr-xr-x | jenkins/build_and_run_docker.sh | 56 | ||||
-rw-r--r-- | jenkins/buildcmds/README.md | 6 | ||||
-rwxr-xr-x | jenkins/buildcmds/pull_request.sh | 15 | ||||
-rw-r--r-- | jenkins/docker/Dockerfile | 130 | ||||
-rw-r--r-- | jenkins/make_test_output.py | 91 | ||||
-rwxr-xr-x | jenkins/pull_request_in_docker.sh | 72 | ||||
-rwxr-xr-x | tests.sh (renamed from travis.sh) | 106 |
15 files changed, 654 insertions, 45 deletions
diff --git a/.travis.yml b/.travis.yml index 60c0427d..bcf3851b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ os: # The Objective C build needs Xcode 7.0 or later. osx_image: xcode7.2 script: - - ./travis.sh $CONFIG + - ./tests.sh $CONFIG env: - CONFIG=cpp - CONFIG=cpp_distcheck diff --git a/csharp/protos/unittest_issues.proto b/csharp/protos/unittest_issues.proto index 989b3dc4..6c9f7634 100644 --- a/csharp/protos/unittest_issues.proto +++ b/csharp/protos/unittest_issues.proto @@ -116,4 +116,11 @@ message TestJsonFieldOrdering { string o2_string = 3; } -}
\ No newline at end of file +} + +message TestJsonName { + // Message for testing the effects for of the json_name option + string name = 1; + string description = 2 [json_name = "desc"]; + string guid = 3 [json_name = "exid"]; +} diff --git a/csharp/src/Google.Protobuf.Test/IssuesTest.cs b/csharp/src/Google.Protobuf.Test/IssuesTest.cs index a0350035..a38d6b08 100644 --- a/csharp/src/Google.Protobuf.Test/IssuesTest.cs +++ b/csharp/src/Google.Protobuf.Test/IssuesTest.cs @@ -59,5 +59,24 @@ namespace Google.Protobuf // Underscores aren't reflected in the JSON.
Assert.AreEqual("{ \"types\": 10, \"descriptor\": 20 }", message.ToString());
}
+
+ [Test]
+ public void JsonNameParseTest()
+ {
+ var settings = new JsonParser.Settings(10, TypeRegistry.FromFiles(UnittestIssuesReflection.Descriptor));
+ var parser = new JsonParser(settings);
+
+ // It is safe to use either original field name or explicitly specified json_name
+ Assert.AreEqual(new TestJsonName { Name = "test", Description = "test2", Guid = "test3" },
+ parser.Parse<TestJsonName>("{ \"name\": \"test\", \"desc\": \"test2\", \"guid\": \"test3\" }"));
+ }
+
+ [Test]
+ public void JsonNameFormatTest()
+ {
+ var message = new TestJsonName { Name = "test", Description = "test2", Guid = "test3" };
+ Assert.AreEqual("{ \"name\": \"test\", \"desc\": \"test2\", \"exid\": \"test3\" }",
+ JsonFormatter.Default.Format(message));
+ }
}
}
diff --git a/csharp/src/Google.Protobuf.Test/TestProtos/UnittestIssues.cs b/csharp/src/Google.Protobuf.Test/TestProtos/UnittestIssues.cs index 16176a33..586f01c8 100644 --- a/csharp/src/Google.Protobuf.Test/TestProtos/UnittestIssues.cs +++ b/csharp/src/Google.Protobuf.Test/TestProtos/UnittestIssues.cs @@ -42,10 +42,12 @@ namespace UnitTest.Issues.TestProtos { "CgtwbGFpbl9pbnQzMhgEIAEoBRITCglvMV9zdHJpbmcYAiABKAlIABISCghv", "MV9pbnQzMhgFIAEoBUgAEhQKDHBsYWluX3N0cmluZxgBIAEoCRISCghvMl9p", "bnQzMhgGIAEoBUgBEhMKCW8yX3N0cmluZxgDIAEoCUgBQgQKAm8xQgQKAm8y", - "KlUKDE5lZ2F0aXZlRW51bRIWChJORUdBVElWRV9FTlVNX1pFUk8QABIWCglG", - "aXZlQmVsb3cQ+///////////ARIVCghNaW51c09uZRD///////////8BKi4K", - "DkRlcHJlY2F0ZWRFbnVtEhMKD0RFUFJFQ0FURURfWkVSTxAAEgcKA29uZRAB", - "Qh9IAaoCGlVuaXRUZXN0Lklzc3Vlcy5UZXN0UHJvdG9zYgZwcm90bzM=")); + "IksKDFRlc3RKc29uTmFtZRIMCgRuYW1lGAEgASgJEhkKC2Rlc2NyaXB0aW9u", + "GAIgASgJUgRkZXNjEhIKBGd1aWQYAyABKAlSBGV4aWQqVQoMTmVnYXRpdmVF", + "bnVtEhYKEk5FR0FUSVZFX0VOVU1fWkVSTxAAEhYKCUZpdmVCZWxvdxD7////", + "//////8BEhUKCE1pbnVzT25lEP///////////wEqLgoORGVwcmVjYXRlZEVu", + "dW0SEwoPREVQUkVDQVRFRF9aRVJPEAASBwoDb25lEAFCH0gBqgIaVW5pdFRl", + "c3QuSXNzdWVzLlRlc3RQcm90b3NiBnByb3RvMw==")); descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, new pbr::FileDescriptor[] { }, new pbr::GeneratedClrTypeInfo(new[] {typeof(global::UnitTest.Issues.TestProtos.NegativeEnum), typeof(global::UnitTest.Issues.TestProtos.DeprecatedEnum), }, new pbr::GeneratedClrTypeInfo[] { @@ -55,7 +57,8 @@ namespace UnitTest.Issues.TestProtos { new pbr::GeneratedClrTypeInfo(typeof(global::UnitTest.Issues.TestProtos.DeprecatedFieldsMessage), global::UnitTest.Issues.TestProtos.DeprecatedFieldsMessage.Parser, new[]{ "PrimitiveValue", "PrimitiveArray", "MessageValue", "MessageArray", "EnumValue", "EnumArray" }, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::UnitTest.Issues.TestProtos.ItemField), global::UnitTest.Issues.TestProtos.ItemField.Parser, new[]{ "Item" }, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::UnitTest.Issues.TestProtos.ReservedNames), global::UnitTest.Issues.TestProtos.ReservedNames.Parser, new[]{ "Types_", "Descriptor_" }, null, null, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(typeof(global::UnitTest.Issues.TestProtos.ReservedNames.Types.SomeNestedType), global::UnitTest.Issues.TestProtos.ReservedNames.Types.SomeNestedType.Parser, null, null, null, null)}), - new pbr::GeneratedClrTypeInfo(typeof(global::UnitTest.Issues.TestProtos.TestJsonFieldOrdering), global::UnitTest.Issues.TestProtos.TestJsonFieldOrdering.Parser, new[]{ "PlainInt32", "O1String", "O1Int32", "PlainString", "O2Int32", "O2String" }, new[]{ "O1", "O2" }, null, null) + new pbr::GeneratedClrTypeInfo(typeof(global::UnitTest.Issues.TestProtos.TestJsonFieldOrdering), global::UnitTest.Issues.TestProtos.TestJsonFieldOrdering.Parser, new[]{ "PlainInt32", "O1String", "O1Int32", "PlainString", "O2Int32", "O2String" }, new[]{ "O1", "O2" }, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::UnitTest.Issues.TestProtos.TestJsonName), global::UnitTest.Issues.TestProtos.TestJsonName.Parser, new[]{ "Name", "Description", "Guid" }, null, null, null) })); } #endregion @@ -1399,6 +1402,166 @@ namespace UnitTest.Issues.TestProtos { } + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + public sealed partial class TestJsonName : pb::IMessage<TestJsonName> { + private static readonly pb::MessageParser<TestJsonName> _parser = new pb::MessageParser<TestJsonName>(() => new TestJsonName()); + public static pb::MessageParser<TestJsonName> Parser { get { return _parser; } } + + public static pbr::MessageDescriptor Descriptor { + get { return global::UnitTest.Issues.TestProtos.UnittestIssuesReflection.Descriptor.MessageTypes[7]; } + } + + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + public TestJsonName() { + OnConstruction(); + } + + partial void OnConstruction(); + + public TestJsonName(TestJsonName other) : this() { + name_ = other.name_; + description_ = other.description_; + guid_ = other.guid_; + } + + public TestJsonName Clone() { + return new TestJsonName(this); + } + + /// <summary>Field number for the "name" field.</summary> + public const int NameFieldNumber = 1; + private string name_ = ""; + /// <summary> + /// json_name field options are not properly handled during deserialization + /// </summary> + public string Name { + get { return name_; } + set { + name_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + + /// <summary>Field number for the "description" field.</summary> + public const int DescriptionFieldNumber = 2; + private string description_ = ""; + public string Description { + get { return description_; } + set { + description_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + + /// <summary>Field number for the "guid" field.</summary> + public const int GuidFieldNumber = 3; + private string guid_ = ""; + public string Guid { + get { return guid_; } + set { + guid_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + + public override bool Equals(object other) { + return Equals(other as TestJsonName); + } + + public bool Equals(TestJsonName other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (Name != other.Name) return false; + if (Description != other.Description) return false; + if (Guid != other.Guid) return false; + return true; + } + + public override int GetHashCode() { + int hash = 1; + if (Name.Length != 0) hash ^= Name.GetHashCode(); + if (Description.Length != 0) hash ^= Description.GetHashCode(); + if (Guid.Length != 0) hash ^= Guid.GetHashCode(); + return hash; + } + + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + public void WriteTo(pb::CodedOutputStream output) { + if (Name.Length != 0) { + output.WriteRawTag(10); + output.WriteString(Name); + } + if (Description.Length != 0) { + output.WriteRawTag(18); + output.WriteString(Description); + } + if (Guid.Length != 0) { + output.WriteRawTag(26); + output.WriteString(Guid); + } + } + + public int CalculateSize() { + int size = 0; + if (Name.Length != 0) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(Name); + } + if (Description.Length != 0) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(Description); + } + if (Guid.Length != 0) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(Guid); + } + return size; + } + + public void MergeFrom(TestJsonName other) { + if (other == null) { + return; + } + if (other.Name.Length != 0) { + Name = other.Name; + } + if (other.Description.Length != 0) { + Description = other.Description; + } + if (other.Guid.Length != 0) { + Guid = other.Guid; + } + } + + public void MergeFrom(pb::CodedInputStream input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + input.SkipLastField(); + break; + case 10: { + Name = input.ReadString(); + break; + } + case 18: { + Description = input.ReadString(); + break; + } + case 26: { + Guid = input.ReadString(); + break; + } + } + } + } + + } + #endregion } diff --git a/csharp/src/Google.Protobuf/JsonFormatter.cs b/csharp/src/Google.Protobuf/JsonFormatter.cs index eb959c9f..cbd9366c 100644 --- a/csharp/src/Google.Protobuf/JsonFormatter.cs +++ b/csharp/src/Google.Protobuf/JsonFormatter.cs @@ -237,11 +237,13 @@ namespace Google.Protobuf { writer.Write(PropertySeparator); } - WriteString(writer, ToCamelCase(accessor.Descriptor.Name)); + + WriteString(writer, accessor.Descriptor.JsonName); writer.Write(NameValueSeparator); WriteValue(writer, value); + first = false; - } + } return !first; } diff --git a/csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs b/csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs index c6caaec6..6083f171 100644 --- a/csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs +++ b/csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs @@ -90,6 +90,12 @@ namespace Google.Protobuf.Reflection /// </summary> public override string Name { get { return proto.Name; } } + + /// <summary> + /// The json_name option of the descriptor's target. + /// </summary> + public string JsonName { get { return proto.JsonName == "" ? JsonFormatter.ToCamelCase(proto.Name) : proto.JsonName; } } + internal FieldDescriptorProto Proto { get { return proto; } } /// <summary> diff --git a/csharp/src/Google.Protobuf/Reflection/MessageDescriptor.cs b/csharp/src/Google.Protobuf/Reflection/MessageDescriptor.cs index f5798d1e..f5a835e5 100644 --- a/csharp/src/Google.Protobuf/Reflection/MessageDescriptor.cs +++ b/csharp/src/Google.Protobuf/Reflection/MessageDescriptor.cs @@ -102,8 +102,8 @@ namespace Google.Protobuf.Reflection var map = new Dictionary<string, FieldDescriptor>(); foreach (var field in fields) { - map[JsonFormatter.ToCamelCase(field.Name)] = field; map[field.Name] = field; + map[field.JsonName] = field; } return new ReadOnlyDictionary<string, FieldDescriptor>(map); } diff --git a/jenkins/README.md b/jenkins/README.md new file mode 100644 index 00000000..29f664f2 --- /dev/null +++ b/jenkins/README.md @@ -0,0 +1,6 @@ + +Jenkins Infrastructure +---------------------- + +The scripts in this directory serve as plumbing for running the protobuf +tests under Jenkins. diff --git a/jenkins/build_and_run_docker.sh b/jenkins/build_and_run_docker.sh new file mode 100755 index 00000000..abc6f055 --- /dev/null +++ b/jenkins/build_and_run_docker.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# +# Builds docker image and runs a command under it. +# This is a generic script that is configured with the following variables: +# +# DOCKERFILE_DIR - Directory in which Dockerfile file is located. +# DOCKER_RUN_SCRIPT - Script to run under docker (relative to protobuf repo root) +# OUTPUT_DIR - Directory that will be copied from inside docker after finishing. +# $@ - Extra args to pass to docker run + + +set -ex + +cd $(dirname $0)/.. +git_root=$(pwd) +cd - + +# Use image name based on Dockerfile location checksum +DOCKER_IMAGE_NAME=$(basename $DOCKERFILE_DIR)_$(sha1sum $DOCKERFILE_DIR/Dockerfile | cut -f1 -d\ ) + +# Make sure docker image has been built. Should be instantaneous if so. +docker build -t $DOCKER_IMAGE_NAME $DOCKERFILE_DIR + +# Ensure existence of ccache directory +CCACHE_DIR=/tmp/protobuf-ccache +mkdir -p $CCACHE_DIR + +# Choose random name for docker container +CONTAINER_NAME="build_and_run_docker_$(uuidgen)" + +# Run command inside docker +docker run \ + "$@" \ + -e CCACHE_DIR=$CCACHE_DIR \ + -e EXTERNAL_GIT_ROOT="/var/local/jenkins/protobuf" \ + -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \ + -v "$git_root:/var/local/jenkins/protobuf:ro" \ + -v $CCACHE_DIR:$CCACHE_DIR \ + -w /var/local/git/protobuf \ + --name=$CONTAINER_NAME \ + $DOCKER_IMAGE_NAME \ + bash -l "/var/local/jenkins/protobuf/$DOCKER_RUN_SCRIPT" || FAILED="true" + +# Copy output artifacts +if [ "$OUTPUT_DIR" != "" ] +then + docker cp "$CONTAINER_NAME:/var/local/git/protobuf/$OUTPUT_DIR" "$git_root" || FAILED="true" +fi + +# remove the container, possibly killing it first +docker rm -f $CONTAINER_NAME || true + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/jenkins/buildcmds/README.md b/jenkins/buildcmds/README.md new file mode 100644 index 00000000..7a48f2d5 --- /dev/null +++ b/jenkins/buildcmds/README.md @@ -0,0 +1,6 @@ + +Jenkins Build Commands +---------------------- + +The scripts in this directory are designed to be top-level entry points for +Jenkins projects. diff --git a/jenkins/buildcmds/pull_request.sh b/jenkins/buildcmds/pull_request.sh new file mode 100755 index 00000000..01fda798 --- /dev/null +++ b/jenkins/buildcmds/pull_request.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# +# This is the top-level script we give to Jenkins as the entry point for +# running the "pull request" project: +# +# https://grpc-testing.appspot.com/view/Protocol%20Buffers/job/protobuf_pull_request/ +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +export DOCKERFILE_DIR=jenkins/docker +export DOCKER_RUN_SCRIPT=jenkins/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +./jenkins/build_and_run_docker.sh diff --git a/jenkins/docker/Dockerfile b/jenkins/docker/Dockerfile new file mode 100644 index 00000000..8467aeff --- /dev/null +++ b/jenkins/docker/Dockerfile @@ -0,0 +1,130 @@ +# This Dockerfile specifies the recipe for creating an image for the tests +# to run in. +# +# We install as many test dependencies here as we can, because these setup +# steps can be cached. They do *not* run every time we run the build. +# The Docker image is only rebuilt when the Dockerfile (ie. this file) +# changes. + +# Base Dockerfile for gRPC dev images +FROM debian:latest + +# Apt source for old Python versions. +RUN echo 'deb http://ppa.launchpad.net/fkrull/deadsnakes/ubuntu trusty main' > /etc/apt/sources.list.d/deadsnakes.list && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys DB82666C + +# Apt source for Oracle Java. +run echo 'deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main' > /etc/apt/sources.list.d/webupd8team-java-trusty.list && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EEA14886 && \ + echo "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections + +# Apt source for Mono +run echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list && \ + echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF + +# Install dependencies. We start with the basic ones require to build protoc +# and the C++ build +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + gcc \ + git \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + parallel \ + time \ + wget \ + # -- For csharp -- + mono-devel \ + referenceassemblies-pcl \ + nunit \ + # -- For all Java builds -- \ + maven \ + # -- For java_jdk6 -- \ + # oops! not in jessie. too old? openjdk-6-jdk \ + # -- For java_jdk7 -- \ + openjdk-7-jdk \ + # -- For java_oracle7 -- \ + oracle-java7-installer \ + # -- For python / python_cpp -- \ + python-setuptools \ + python-pip \ + python-dev \ + python2.6-dev \ + python3.3-dev \ + python3.4-dev \ + # -- For Ruby -- + ruby \ + && apt-get clean + +################## +# C# dependencies + +RUN wget www.nuget.org/NuGet.exe -O /usr/local/bin/nuget.exe + +################## +# Python dependencies + +# These packages exist in apt-get, but their versions are too old, so we have +# to get updates from pip. + +RUN pip install pip --upgrade +RUN pip install virtualenv tox yattag + + +################## +# Ruby dependencies + +# Install rvm +RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 +RUN \curl -sSL https://get.rvm.io | bash -s stable + +# Install Ruby 2.1 +RUN /bin/bash -l -c "rvm install ruby-2.1" +RUN /bin/bash -l -c "rvm use --default ruby-2.1" +RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc" +RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" +RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc" +RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" + +################## +# Java dependencies + +# This step requires compiling protoc. :( + +ENV MAVEN_REPO /var/maven_local_repository +ENV MVN mvn --batch-mode + +RUN cd /tmp && \ + git clone https://github.com/google/protobuf.git && \ + cd protobuf && \ + ./autogen.sh && \ + ./configure && \ + make -j6 && \ + cd java && \ + $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO -P lite && \ + $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO && \ + cd ../javanano && \ + $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO + +################## +# Prepare ccache + +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + +# Define the default command. +CMD ["bash"] diff --git a/jenkins/make_test_output.py b/jenkins/make_test_output.py new file mode 100644 index 00000000..b1f2e2c0 --- /dev/null +++ b/jenkins/make_test_output.py @@ -0,0 +1,91 @@ +"""Gathers output from test runs and create an XML file in JUnit format. + +The output files from the individual tests have been written in a directory +structure like: + + $DIR/joblog (output from "parallel --joblog joblog") + $DIR/logs/1/cpp/stdout + $DIR/logs/1/cpp/stderr + $DIR/logs/1/csharp/stdout + $DIR/logs/1/csharp/stderr + $DIR/logs/1/java_jdk7/stdout + $DIR/logs/1/java_jdk7/stderr + etc. + +This script bundles them into a single output XML file so Jenkins can show +detailed test results. It runs as the last step before the Jenkins build +finishes. +""" + +import os; +import sys; +from yattag import Doc +from collections import defaultdict + +def readtests(basedir): + tests = defaultdict(dict) + + # Sample input (note: separators are tabs). + # + # Seq Host Starttime Runtime Send Receive Exitval Signal Command + # 1 : 1456263838.313 0.005 0 0 0 0 echo A + with open(basedir + "/joblog") as jobs: + firstline = next(jobs) + for line in jobs: + values = line.split("\t") + + name = values[8].split()[-1] + test = tests[name] + test["name"] = name + test["time"] = values[3] + + exitval = values[6] + if int(exitval): + # We don't have a more specific message. User should look at stderr. + test["failure"] = "TEST FAILURE" + else: + test["failure"] = False + + for testname in os.listdir(basedir + "/logs/1"): + test = tests[testname] + + with open(basedir + "/logs/1/" + testname + "/stdout") as f: + test["stdout"] = f.read() + + with open(basedir + "/logs/1/" + testname + "/stderr") as f: + test["stderr"] = f.read() + + # The cpp test is special since it doesn't run under parallel so doesn't show + # up in the job log. + tests["cpp"]["name"] = "cpp" + + with open(basedir + '/logs/1/cpp/build_time', 'r') as f: + tests["cpp"]["time"] = f.read().strip() + tests["cpp"]["failure"] = False + + ret = tests.values() + ret.sort(key=lambda x: x["name"]) + + return ret + +def genxml(tests): + doc, tag, text = Doc().tagtext() + + with tag("testsuites"): + with tag("testsuite", name="Protobuf Tests"): + for test in tests: + with tag("testcase", name=test["name"], classname=test["name"], + time=test["time"]): + with tag("system-out"): + text(test["stdout"]) + with tag("system-err"): + text(test["stderr"]) + if test["failure"]: + with tag("failure"): + text(test["failure"]) + + return doc.getvalue() + +sys.stderr.write("make_test_output.py: writing XML from directory: " + + sys.argv[1] + "\n"); +print genxml(readtests(sys.argv[1])) diff --git a/jenkins/pull_request_in_docker.sh b/jenkins/pull_request_in_docker.sh new file mode 100755 index 00000000..887f97c5 --- /dev/null +++ b/jenkins/pull_request_in_docker.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# +# This is the script that runs inside Docker, once the image has been built, +# to execute all tests for the "pull request" project. + +WORKSPACE_BASE=`pwd` +MY_DIR="$(dirname "$0")" +TEST_SCRIPT=$MY_DIR/../tests.sh +BUILD_DIR=/tmp/protobuf + +set -e # exit immediately on error +set -x # display all commands + +# The protobuf repository is mounted into our Docker image, but read-only. +# We clone into a directory inside Docker (this is faster than cp). +rm -rf $BUILD_DIR +mkdir -p $BUILD_DIR +cd $BUILD_DIR +git clone /var/local/jenkins/protobuf +cd protobuf + +# Set up the directory where our test output is going to go. +OUTPUT_DIR=`mktemp -d` +LOG_OUTPUT_DIR=$OUTPUT_DIR/logs +mkdir -p $LOG_OUTPUT_DIR/1/cpp + +################################################################################ +# cpp build needs to run first, non-parallelized, so that protoc is available +# for other builds. + +# Output filenames to follow the overall scheme used by parallel, ie: +# $DIR/logs/1/cpp/stdout +# $DIR/logs/1/cpp/stderr +# $DIR/logs/1/csharp/stdout +# $DIR/logs/1/csharp/stderr +# $DIR/logs/1/java_jdk7/stdout +# $DIR/logs/1/java_jdk7/stderr +CPP_STDOUT=$LOG_OUTPUT_DIR/1/cpp/stdout +CPP_STDERR=$LOG_OUTPUT_DIR/1/cpp/stderr + +# Time the C++ build, so we can put this info in the test output. +# It's important that we get /usr/bin/time (which supports -f and -o) and not +# the bash builtin "time" which doesn't. +TIME_CMD="/usr/bin/time -f %e -o $LOG_OUTPUT_DIR/1/cpp/build_time" + +$TIME_CMD $TEST_SCRIPT cpp > >(tee $CPP_STDOUT) 2> >(tee $CPP_STDERR >&2) + +# Other tests are run in parallel. + +parallel --results $LOG_OUTPUT_DIR --joblog $OUTPUT_DIR/joblog $TEST_SCRIPT ::: \ + csharp \ + java_jdk7 \ + javanano_jdk7 \ + java_oracle7 \ + javanano_oracle7 \ + python \ + python_cpp \ + ruby21 \ + || true # Process test results even if tests fail. + +cat $OUTPUT_DIR/joblog + +# The directory that is copied from Docker back into the Jenkins workspace. +COPY_FROM_DOCKER=/var/local/git/protobuf/testoutput +mkdir -p $COPY_FROM_DOCKER +TESTOUTPUT_XML_FILE=$COPY_FROM_DOCKER/testresults.xml + +# Process all the output files from "parallel" and package them into a single +# .xml file with detailed, broken-down test output. +python $MY_DIR/make_test_output.py $OUTPUT_DIR > $TESTOUTPUT_XML_FILE + +ls -l $TESTOUTPUT_XML_FILE @@ -1,17 +1,24 @@ -#!/usr/bin/env bash +#!/bin/bash +# +# Build and runs tests for the protobuf project. The tests as written here are +# used by both Jenkins and Travis, though some specialized logic is required to +# handle the differences between them. -# Note: travis currently does not support testing more than one language so the -# .travis.yml cheats and claims to only be cpp. If they add multiple language -# support, this should probably get updated to install steps and/or -# rvm/gemfile/jdk/etc. entries rather than manually doing the work. - -# .travis.yml uses matrix.exclude to block the cases where app-get can't be -# use to install things. +on_travis() { + if [ "$TRAVIS" == "true" ]; then + "$@" + fi +} # For when some other test needs the C++ main build, including protoc and # libprotobuf. internal_build_cpp() { - if [ $(uname -s) == "Linux" ]; then + if [ -f src/protoc ]; then + # Already built. + return + fi + + if [[ $(uname -s) == "Linux" && "$TRAVIS" == "true" ]]; then # Install GCC 4.8 to replace the default GCC 4.6. We need 4.8 for more # decent C++ 11 support in order to compile conformance tests. sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y @@ -42,16 +49,20 @@ build_csharp() { # need to really build protoc, but it's simplest to keep with the # conventions of the other builds. internal_build_cpp + NUGET=/usr/local/bin/nuget.exe - # Install latest version of Mono - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF - echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list - echo "deb http://download.mono-project.com/repo/debian wheezy-libtiff-compat main" | sudo tee -a /etc/apt/sources.list.d/mono-xamarin.list - sudo apt-get update -qq - sudo apt-get install -qq mono-devel referenceassemblies-pcl nunit - wget www.nuget.org/NuGet.exe -O nuget.exe + if [ "$TRAVIS" == "true" ]; then + # Install latest version of Mono + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF + echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list + echo "deb http://download.mono-project.com/repo/debian wheezy-libtiff-compat main" | sudo tee -a /etc/apt/sources.list.d/mono-xamarin.list + sudo apt-get update -qq + sudo apt-get install -qq mono-devel referenceassemblies-pcl nunit + wget www.nuget.org/NuGet.exe -O nuget.exe + NUGET=../../nuget.exe + fi - (cd csharp/src; mono ../../nuget.exe restore) + (cd csharp/src; mono $NUGET restore) csharp/buildall.sh cd conformance && make test_csharp && cd .. } @@ -78,40 +89,54 @@ use_java() { version=$1 case "$version" in jdk6) - sudo apt-get install openjdk-6-jdk + on_travis sudo apt-get install openjdk-6-jdk export PATH=/usr/lib/jvm/java-6-openjdk-amd64/bin:$PATH ;; jdk7) - sudo apt-get install openjdk-7-jdk + on_travis sudo apt-get install openjdk-7-jdk export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH ;; oracle7) - sudo apt-get install python-software-properties # for apt-add-repository - echo "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" | \ - sudo debconf-set-selections - yes | sudo apt-add-repository ppa:webupd8team/java - yes | sudo apt-get install oracle-java7-installer + if [ "$TRAVIS" == "true" ]; then + sudo apt-get install python-software-properties # for apt-add-repository + echo "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" | \ + sudo debconf-set-selections + yes | sudo apt-add-repository ppa:webupd8team/java + yes | sudo apt-get install oracle-java7-installer + fi; export PATH=/usr/lib/jvm/java-7-oracle/bin:$PATH ;; esac + if [ "$TRAVIS" != "true" ]; then + MAVEN_LOCAL_REPOSITORY=/var/maven_local_repository + MVN="$MVN -e -X --offline -Dmaven.repo.local=$MAVEN_LOCAL_REPOSITORY" + fi; + which java java -version } +# --batch-mode supresses download progress output that spams the logs. +MVN="mvn --batch-mode" + build_java() { + version=$1 + dir=java_$version # Java build needs `protoc`. internal_build_cpp - cd java && mvn test && mvn install - cd util && mvn test + cp -r java $dir + cd $dir && $MVN clean && $MVN test cd ../.. } +# The conformance tests are hard-coded to work with the $ROOT/java directory. +# So this can't run in parallel with two different sets of tests. build_java_with_conformance_tests() { # Java build needs `protoc`. internal_build_cpp - cd java && mvn test && mvn install - cd util && mvn test && mvn assembly:single + cd java && $MVN test && $MVN install + cd util && $MVN package assembly:single cd ../.. cd conformance && make test_java && cd .. } @@ -119,12 +144,12 @@ build_java_with_conformance_tests() { build_javanano() { # Java build needs `protoc`. internal_build_cpp - cd javanano && mvn test && cd .. + cd javanano && $MVN test && cd .. } build_java_jdk6() { use_java jdk6 - build_java + build_java jdk6 } build_java_jdk7() { use_java jdk7 @@ -132,7 +157,7 @@ build_java_jdk7() { } build_java_oracle7() { use_java oracle7 - build_java + build_java oracle7 } build_javanano_jdk6() { @@ -149,6 +174,9 @@ build_javanano_oracle7() { } internal_install_python_deps() { + if [ "$TRAVIS" != "true" ]; then + return; + fi # Install tox (OS X doesn't have pip). if [ $(uname -s) == "Darwin" ]; then sudo easy_install tox @@ -278,6 +306,14 @@ build_javascript() { cd js && npm install && npm test && cd .. } +# Note: travis currently does not support testing more than one language so the +# .travis.yml cheats and claims to only be cpp. If they add multiple language +# support, this should probably get updated to install steps and/or +# rvm/gemfile/jdk/etc. entries rather than manually doing the work. + +# .travis.yml uses matrix.exclude to block the cases where app-get can't be +# use to install things. + # -------- main -------- if [ "$#" -ne 1 ]; then @@ -294,10 +330,10 @@ Usage: $0 { cpp | objectivec_osx | python | python_cpp | - ruby_19 | - ruby_20 | - ruby_21 | - ruby_22 | + ruby19 | + ruby20 | + ruby21 | + ruby22 | jruby } " exit 1 |