diff options
-rw-r--r-- | .travis.yml | 2 | ||||
-rw-r--r-- | csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs | 100 | ||||
-rw-r--r-- | jenkins/README.md | 6 | ||||
-rwxr-xr-x | jenkins/build_and_run_docker.sh | 56 | ||||
-rw-r--r-- | jenkins/buildcmds/README.md | 6 | ||||
-rwxr-xr-x | jenkins/buildcmds/pull_request.sh | 15 | ||||
-rw-r--r-- | jenkins/docker/Dockerfile | 130 | ||||
-rw-r--r-- | jenkins/make_test_output.py | 91 | ||||
-rwxr-xr-x | jenkins/pull_request_in_docker.sh | 72 | ||||
-rwxr-xr-x | python/google/protobuf/internal/message_test.py | 59 | ||||
-rw-r--r-- | python/google/protobuf/pyext/message.cc | 46 | ||||
-rw-r--r-- | src/google/protobuf/stubs/int128.cc | 2 | ||||
-rw-r--r-- | src/google/protobuf/stubs/statusor.h | 2 | ||||
-rwxr-xr-x | tests.sh (renamed from travis.sh) | 106 |
14 files changed, 583 insertions, 110 deletions
diff --git a/.travis.yml b/.travis.yml index 60c0427d..bcf3851b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ os: # The Objective C build needs Xcode 7.0 or later. osx_image: xcode7.2 script: - - ./travis.sh $CONFIG + - ./tests.sh $CONFIG env: - CONFIG=cpp - CONFIG=cpp_distcheck diff --git a/csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs b/csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs index 6083f171..de6e5717 100644 --- a/csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs +++ b/csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs @@ -30,9 +30,8 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endregion -using System; -using System.Linq; using Google.Protobuf.Compatibility; +using System; namespace Google.Protobuf.Reflection { @@ -41,20 +40,35 @@ namespace Google.Protobuf.Reflection /// </summary> public sealed class FieldDescriptor : DescriptorBase, IComparable<FieldDescriptor> { - private readonly FieldDescriptorProto proto; private EnumDescriptor enumType; private MessageDescriptor messageType; - private readonly MessageDescriptor containingType; - private readonly OneofDescriptor containingOneof; private FieldType fieldType; private readonly string propertyName; // Annoyingly, needed in Crosslink. private IFieldAccessor accessor; + /// <summary> + /// Get the field's containing message type. + /// </summary> + public MessageDescriptor ContainingType { get; } + + /// <summary> + /// Returns the oneof containing this field, or <c>null</c> if it is not part of a oneof. + /// </summary> + public OneofDescriptor ContainingOneof { get; } + + /// <summary> + /// The effective JSON name for this field. This is usually the lower-camel-cased form of the field name, + /// but can be overridden using the <c>json_name</c> option in the .proto file. + /// </summary> + public string JsonName { get; } + + internal FieldDescriptorProto Proto { get; } + internal FieldDescriptor(FieldDescriptorProto proto, FileDescriptor file, MessageDescriptor parent, int index, string propertyName) : base(file, file.ComputeFullName(parent, proto.Name), index) { - this.proto = proto; + Proto = proto; if (proto.Type != 0) { fieldType = GetFieldTypeFromProtoType(proto.Type); @@ -64,7 +78,7 @@ namespace Google.Protobuf.Reflection { throw new DescriptorValidationException(this, "Field numbers must be positive integers."); } - containingType = parent; + ContainingType = parent; // OneofIndex "defaults" to -1 due to a hack in FieldDescriptor.OnConstruction. if (proto.OneofIndex != -1) { @@ -73,7 +87,7 @@ namespace Google.Protobuf.Reflection throw new DescriptorValidationException(this, $"FieldDescriptorProto.oneof_index is out of range for type {parent.Name}"); } - containingOneof = parent.Oneofs[proto.OneofIndex]; + ContainingOneof = parent.Oneofs[proto.OneofIndex]; } file.DescriptorPool.AddSymbol(this); @@ -83,20 +97,14 @@ namespace Google.Protobuf.Reflection // We could trust the generated code and check whether the type of the property is // a MapField, but that feels a tad nasty. this.propertyName = propertyName; + JsonName = Proto.JsonName == "" ? JsonFormatter.ToCamelCase(Proto.Name) : Proto.JsonName; } + /// <summary> /// The brief name of the descriptor's target. /// </summary> - public override string Name { get { return proto.Name; } } - - - /// <summary> - /// The json_name option of the descriptor's target. - /// </summary> - public string JsonName { get { return proto.JsonName == "" ? JsonFormatter.ToCamelCase(proto.Name) : proto.JsonName; } } - - internal FieldDescriptorProto Proto { get { return proto; } } + public override string Name => Proto.Name; /// <summary> /// Returns the accessor for this field. @@ -116,7 +124,7 @@ namespace Google.Protobuf.Reflection /// and this property will return null. /// </para> /// </remarks> - public IFieldAccessor Accessor { get { return accessor; } } + public IFieldAccessor Accessor => accessor; /// <summary> /// Maps a field type as included in the .proto file to a FieldType. @@ -169,62 +177,32 @@ namespace Google.Protobuf.Reflection /// <summary> /// Returns <c>true</c> if this field is a repeated field; <c>false</c> otherwise. /// </summary> - public bool IsRepeated - { - get { return Proto.Label == FieldDescriptorProto.Types.Label.LABEL_REPEATED; } - } + public bool IsRepeated => Proto.Label == FieldDescriptorProto.Types.Label.LABEL_REPEATED; /// <summary> /// Returns <c>true</c> if this field is a map field; <c>false</c> otherwise. /// </summary> - public bool IsMap - { - get { return fieldType == FieldType.Message && messageType.Proto.Options != null && messageType.Proto.Options.MapEntry; } - } + public bool IsMap => fieldType == FieldType.Message && messageType.Proto.Options != null && messageType.Proto.Options.MapEntry; /// <summary> /// Returns <c>true</c> if this field is a packed, repeated field; <c>false</c> otherwise. /// </summary> - public bool IsPacked - { + public bool IsPacked => // Note the || rather than && here - we're effectively defaulting to packed, because that *is* // the default in proto3, which is all we support. We may give the wrong result for the protos // within descriptor.proto, but that's okay, as they're never exposed and we don't use IsPacked // within the runtime. - get { return Proto.Options == null || Proto.Options.Packed; } - } - - /// <summary> - /// Get the field's containing message type. - /// </summary> - public MessageDescriptor ContainingType - { - get { return containingType; } - } - - /// <summary> - /// Returns the oneof containing this field, or <c>null</c> if it is not part of a oneof. - /// </summary> - public OneofDescriptor ContainingOneof - { - get { return containingOneof; } - } - + Proto.Options == null || Proto.Options.Packed; + /// <summary> /// Returns the type of the field. /// </summary> - public FieldType FieldType - { - get { return fieldType; } - } + public FieldType FieldType => fieldType; /// <summary> /// Returns the field number declared in the proto file. /// </summary> - public int FieldNumber - { - get { return Proto.Number; } - } + public int FieldNumber => Proto.Number; /// <summary> /// Compares this descriptor with another one, ordering in "canonical" order @@ -234,7 +212,7 @@ namespace Google.Protobuf.Reflection /// </summary> public int CompareTo(FieldDescriptor other) { - if (other.containingType != containingType) + if (other.ContainingType != ContainingType) { throw new ArgumentException("FieldDescriptors can only be compared to other FieldDescriptors " + "for fields of the same message type."); @@ -337,14 +315,14 @@ namespace Google.Protobuf.Reflection File.DescriptorPool.AddFieldByNumber(this); - if (containingType != null && containingType.Proto.Options != null && containingType.Proto.Options.MessageSetWireFormat) + if (ContainingType != null && ContainingType.Proto.Options != null && ContainingType.Proto.Options.MessageSetWireFormat) { throw new DescriptorValidationException(this, "MessageSet format is not supported."); } - accessor = CreateAccessor(propertyName); + accessor = CreateAccessor(); } - private IFieldAccessor CreateAccessor(string propertyName) + private IFieldAccessor CreateAccessor() { // If we're given no property name, that's because we really don't want an accessor. // (At the moment, that means it's a map entry message...) @@ -352,10 +330,10 @@ namespace Google.Protobuf.Reflection { return null; } - var property = containingType.ClrType.GetProperty(propertyName); + var property = ContainingType.ClrType.GetProperty(propertyName); if (property == null) { - throw new DescriptorValidationException(this, $"Property {propertyName} not found in {containingType.ClrType}"); + throw new DescriptorValidationException(this, $"Property {propertyName} not found in {ContainingType.ClrType}"); } return IsMap ? new MapFieldAccessor(property, this) : IsRepeated ? new RepeatedFieldAccessor(property, this) diff --git a/jenkins/README.md b/jenkins/README.md new file mode 100644 index 00000000..29f664f2 --- /dev/null +++ b/jenkins/README.md @@ -0,0 +1,6 @@ + +Jenkins Infrastructure +---------------------- + +The scripts in this directory serve as plumbing for running the protobuf +tests under Jenkins. diff --git a/jenkins/build_and_run_docker.sh b/jenkins/build_and_run_docker.sh new file mode 100755 index 00000000..abc6f055 --- /dev/null +++ b/jenkins/build_and_run_docker.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# +# Builds docker image and runs a command under it. +# This is a generic script that is configured with the following variables: +# +# DOCKERFILE_DIR - Directory in which Dockerfile file is located. +# DOCKER_RUN_SCRIPT - Script to run under docker (relative to protobuf repo root) +# OUTPUT_DIR - Directory that will be copied from inside docker after finishing. +# $@ - Extra args to pass to docker run + + +set -ex + +cd $(dirname $0)/.. +git_root=$(pwd) +cd - + +# Use image name based on Dockerfile location checksum +DOCKER_IMAGE_NAME=$(basename $DOCKERFILE_DIR)_$(sha1sum $DOCKERFILE_DIR/Dockerfile | cut -f1 -d\ ) + +# Make sure docker image has been built. Should be instantaneous if so. +docker build -t $DOCKER_IMAGE_NAME $DOCKERFILE_DIR + +# Ensure existence of ccache directory +CCACHE_DIR=/tmp/protobuf-ccache +mkdir -p $CCACHE_DIR + +# Choose random name for docker container +CONTAINER_NAME="build_and_run_docker_$(uuidgen)" + +# Run command inside docker +docker run \ + "$@" \ + -e CCACHE_DIR=$CCACHE_DIR \ + -e EXTERNAL_GIT_ROOT="/var/local/jenkins/protobuf" \ + -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \ + -v "$git_root:/var/local/jenkins/protobuf:ro" \ + -v $CCACHE_DIR:$CCACHE_DIR \ + -w /var/local/git/protobuf \ + --name=$CONTAINER_NAME \ + $DOCKER_IMAGE_NAME \ + bash -l "/var/local/jenkins/protobuf/$DOCKER_RUN_SCRIPT" || FAILED="true" + +# Copy output artifacts +if [ "$OUTPUT_DIR" != "" ] +then + docker cp "$CONTAINER_NAME:/var/local/git/protobuf/$OUTPUT_DIR" "$git_root" || FAILED="true" +fi + +# remove the container, possibly killing it first +docker rm -f $CONTAINER_NAME || true + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/jenkins/buildcmds/README.md b/jenkins/buildcmds/README.md new file mode 100644 index 00000000..7a48f2d5 --- /dev/null +++ b/jenkins/buildcmds/README.md @@ -0,0 +1,6 @@ + +Jenkins Build Commands +---------------------- + +The scripts in this directory are designed to be top-level entry points for +Jenkins projects. diff --git a/jenkins/buildcmds/pull_request.sh b/jenkins/buildcmds/pull_request.sh new file mode 100755 index 00000000..01fda798 --- /dev/null +++ b/jenkins/buildcmds/pull_request.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# +# This is the top-level script we give to Jenkins as the entry point for +# running the "pull request" project: +# +# https://grpc-testing.appspot.com/view/Protocol%20Buffers/job/protobuf_pull_request/ +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +export DOCKERFILE_DIR=jenkins/docker +export DOCKER_RUN_SCRIPT=jenkins/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +./jenkins/build_and_run_docker.sh diff --git a/jenkins/docker/Dockerfile b/jenkins/docker/Dockerfile new file mode 100644 index 00000000..8467aeff --- /dev/null +++ b/jenkins/docker/Dockerfile @@ -0,0 +1,130 @@ +# This Dockerfile specifies the recipe for creating an image for the tests +# to run in. +# +# We install as many test dependencies here as we can, because these setup +# steps can be cached. They do *not* run every time we run the build. +# The Docker image is only rebuilt when the Dockerfile (ie. this file) +# changes. + +# Base Dockerfile for gRPC dev images +FROM debian:latest + +# Apt source for old Python versions. +RUN echo 'deb http://ppa.launchpad.net/fkrull/deadsnakes/ubuntu trusty main' > /etc/apt/sources.list.d/deadsnakes.list && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys DB82666C + +# Apt source for Oracle Java. +run echo 'deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main' > /etc/apt/sources.list.d/webupd8team-java-trusty.list && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EEA14886 && \ + echo "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections + +# Apt source for Mono +run echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list && \ + echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF + +# Install dependencies. We start with the basic ones require to build protoc +# and the C++ build +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + gcc \ + git \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + parallel \ + time \ + wget \ + # -- For csharp -- + mono-devel \ + referenceassemblies-pcl \ + nunit \ + # -- For all Java builds -- \ + maven \ + # -- For java_jdk6 -- \ + # oops! not in jessie. too old? openjdk-6-jdk \ + # -- For java_jdk7 -- \ + openjdk-7-jdk \ + # -- For java_oracle7 -- \ + oracle-java7-installer \ + # -- For python / python_cpp -- \ + python-setuptools \ + python-pip \ + python-dev \ + python2.6-dev \ + python3.3-dev \ + python3.4-dev \ + # -- For Ruby -- + ruby \ + && apt-get clean + +################## +# C# dependencies + +RUN wget www.nuget.org/NuGet.exe -O /usr/local/bin/nuget.exe + +################## +# Python dependencies + +# These packages exist in apt-get, but their versions are too old, so we have +# to get updates from pip. + +RUN pip install pip --upgrade +RUN pip install virtualenv tox yattag + + +################## +# Ruby dependencies + +# Install rvm +RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 +RUN \curl -sSL https://get.rvm.io | bash -s stable + +# Install Ruby 2.1 +RUN /bin/bash -l -c "rvm install ruby-2.1" +RUN /bin/bash -l -c "rvm use --default ruby-2.1" +RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc" +RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" +RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc" +RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" + +################## +# Java dependencies + +# This step requires compiling protoc. :( + +ENV MAVEN_REPO /var/maven_local_repository +ENV MVN mvn --batch-mode + +RUN cd /tmp && \ + git clone https://github.com/google/protobuf.git && \ + cd protobuf && \ + ./autogen.sh && \ + ./configure && \ + make -j6 && \ + cd java && \ + $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO -P lite && \ + $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO && \ + cd ../javanano && \ + $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO + +################## +# Prepare ccache + +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + +# Define the default command. +CMD ["bash"] diff --git a/jenkins/make_test_output.py b/jenkins/make_test_output.py new file mode 100644 index 00000000..b1f2e2c0 --- /dev/null +++ b/jenkins/make_test_output.py @@ -0,0 +1,91 @@ +"""Gathers output from test runs and create an XML file in JUnit format. + +The output files from the individual tests have been written in a directory +structure like: + + $DIR/joblog (output from "parallel --joblog joblog") + $DIR/logs/1/cpp/stdout + $DIR/logs/1/cpp/stderr + $DIR/logs/1/csharp/stdout + $DIR/logs/1/csharp/stderr + $DIR/logs/1/java_jdk7/stdout + $DIR/logs/1/java_jdk7/stderr + etc. + +This script bundles them into a single output XML file so Jenkins can show +detailed test results. It runs as the last step before the Jenkins build +finishes. +""" + +import os; +import sys; +from yattag import Doc +from collections import defaultdict + +def readtests(basedir): + tests = defaultdict(dict) + + # Sample input (note: separators are tabs). + # + # Seq Host Starttime Runtime Send Receive Exitval Signal Command + # 1 : 1456263838.313 0.005 0 0 0 0 echo A + with open(basedir + "/joblog") as jobs: + firstline = next(jobs) + for line in jobs: + values = line.split("\t") + + name = values[8].split()[-1] + test = tests[name] + test["name"] = name + test["time"] = values[3] + + exitval = values[6] + if int(exitval): + # We don't have a more specific message. User should look at stderr. + test["failure"] = "TEST FAILURE" + else: + test["failure"] = False + + for testname in os.listdir(basedir + "/logs/1"): + test = tests[testname] + + with open(basedir + "/logs/1/" + testname + "/stdout") as f: + test["stdout"] = f.read() + + with open(basedir + "/logs/1/" + testname + "/stderr") as f: + test["stderr"] = f.read() + + # The cpp test is special since it doesn't run under parallel so doesn't show + # up in the job log. + tests["cpp"]["name"] = "cpp" + + with open(basedir + '/logs/1/cpp/build_time', 'r') as f: + tests["cpp"]["time"] = f.read().strip() + tests["cpp"]["failure"] = False + + ret = tests.values() + ret.sort(key=lambda x: x["name"]) + + return ret + +def genxml(tests): + doc, tag, text = Doc().tagtext() + + with tag("testsuites"): + with tag("testsuite", name="Protobuf Tests"): + for test in tests: + with tag("testcase", name=test["name"], classname=test["name"], + time=test["time"]): + with tag("system-out"): + text(test["stdout"]) + with tag("system-err"): + text(test["stderr"]) + if test["failure"]: + with tag("failure"): + text(test["failure"]) + + return doc.getvalue() + +sys.stderr.write("make_test_output.py: writing XML from directory: " + + sys.argv[1] + "\n"); +print genxml(readtests(sys.argv[1])) diff --git a/jenkins/pull_request_in_docker.sh b/jenkins/pull_request_in_docker.sh new file mode 100755 index 00000000..887f97c5 --- /dev/null +++ b/jenkins/pull_request_in_docker.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# +# This is the script that runs inside Docker, once the image has been built, +# to execute all tests for the "pull request" project. + +WORKSPACE_BASE=`pwd` +MY_DIR="$(dirname "$0")" +TEST_SCRIPT=$MY_DIR/../tests.sh +BUILD_DIR=/tmp/protobuf + +set -e # exit immediately on error +set -x # display all commands + +# The protobuf repository is mounted into our Docker image, but read-only. +# We clone into a directory inside Docker (this is faster than cp). +rm -rf $BUILD_DIR +mkdir -p $BUILD_DIR +cd $BUILD_DIR +git clone /var/local/jenkins/protobuf +cd protobuf + +# Set up the directory where our test output is going to go. +OUTPUT_DIR=`mktemp -d` +LOG_OUTPUT_DIR=$OUTPUT_DIR/logs +mkdir -p $LOG_OUTPUT_DIR/1/cpp + +################################################################################ +# cpp build needs to run first, non-parallelized, so that protoc is available +# for other builds. + +# Output filenames to follow the overall scheme used by parallel, ie: +# $DIR/logs/1/cpp/stdout +# $DIR/logs/1/cpp/stderr +# $DIR/logs/1/csharp/stdout +# $DIR/logs/1/csharp/stderr +# $DIR/logs/1/java_jdk7/stdout +# $DIR/logs/1/java_jdk7/stderr +CPP_STDOUT=$LOG_OUTPUT_DIR/1/cpp/stdout +CPP_STDERR=$LOG_OUTPUT_DIR/1/cpp/stderr + +# Time the C++ build, so we can put this info in the test output. +# It's important that we get /usr/bin/time (which supports -f and -o) and not +# the bash builtin "time" which doesn't. +TIME_CMD="/usr/bin/time -f %e -o $LOG_OUTPUT_DIR/1/cpp/build_time" + +$TIME_CMD $TEST_SCRIPT cpp > >(tee $CPP_STDOUT) 2> >(tee $CPP_STDERR >&2) + +# Other tests are run in parallel. + +parallel --results $LOG_OUTPUT_DIR --joblog $OUTPUT_DIR/joblog $TEST_SCRIPT ::: \ + csharp \ + java_jdk7 \ + javanano_jdk7 \ + java_oracle7 \ + javanano_oracle7 \ + python \ + python_cpp \ + ruby21 \ + || true # Process test results even if tests fail. + +cat $OUTPUT_DIR/joblog + +# The directory that is copied from Docker back into the Jenkins workspace. +COPY_FROM_DOCKER=/var/local/git/protobuf/testoutput +mkdir -p $COPY_FROM_DOCKER +TESTOUTPUT_XML_FILE=$COPY_FROM_DOCKER/testresults.xml + +# Process all the output files from "parallel" and package them into a single +# .xml file with detailed, broken-down test output. +python $MY_DIR/make_test_output.py $OUTPUT_DIR > $TESTOUTPUT_XML_FILE + +ls -l $TESTOUTPUT_XML_FILE diff --git a/python/google/protobuf/internal/message_test.py b/python/google/protobuf/internal/message_test.py index d03f2d25..ee6b944a 100755 --- a/python/google/protobuf/internal/message_test.py +++ b/python/google/protobuf/internal/message_test.py @@ -57,7 +57,11 @@ try: except ImportError: import unittest from google.protobuf.internal import _parameterized +from google.protobuf import descriptor_pb2 +from google.protobuf import descriptor_pool from google.protobuf import map_unittest_pb2 +from google.protobuf import message_factory +from google.protobuf import text_format from google.protobuf import unittest_pb2 from google.protobuf import unittest_proto3_arena_pb2 from google.protobuf.internal import any_test_pb2 @@ -1776,5 +1780,60 @@ class PackedFieldTest(unittest.TestCase): b'\x70\x01') self.assertEqual(golden_data, message.SerializeToString()) + +@unittest.skipIf(api_implementation.Type() != 'cpp', + 'explicit tests of the C++ implementation') +class OversizeProtosTest(unittest.TestCase): + + def setUp(self): + self.file_desc = """ + name: "f/f.msg2" + package: "f" + message_type { + name: "msg1" + field { + name: "payload" + number: 1 + label: LABEL_OPTIONAL + type: TYPE_STRING + } + } + message_type { + name: "msg2" + field { + name: "field" + number: 1 + label: LABEL_OPTIONAL + type: TYPE_MESSAGE + type_name: "msg1" + } + } + """ + pool = descriptor_pool.DescriptorPool() + desc = descriptor_pb2.FileDescriptorProto() + text_format.Parse(self.file_desc, desc) + pool.Add(desc) + self.proto_cls = message_factory.MessageFactory(pool).GetPrototype( + pool.FindMessageTypeByName('f.msg2')) + self.p = self.proto_cls() + self.p.field.payload = 'c' * (1024 * 1024 * 64 + 1) + self.p_serialized = self.p.SerializeToString() + + def testAssertOversizeProto(self): + from google.protobuf.pyext._message import SetAllowOversizeProtos + SetAllowOversizeProtos(False) + q = self.proto_cls() + try: + q.ParseFromString(self.p_serialized) + except message.DecodeError as e: + self.assertEqual(str(e), 'Error parsing message') + + def testSucceedOversizeProto(self): + from google.protobuf.pyext._message import SetAllowOversizeProtos + SetAllowOversizeProtos(True) + q = self.proto_cls() + q.ParseFromString(self.p_serialized) + self.assertEqual(self.p.field.payload, q.field.payload) + if __name__ == '__main__': unittest.main() diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc index 60ec9c1b..7d3e8c37 100644 --- a/python/google/protobuf/pyext/message.cc +++ b/python/google/protobuf/pyext/message.cc @@ -1911,6 +1911,30 @@ static PyObject* CopyFrom(CMessage* self, PyObject* arg) { Py_RETURN_NONE; } +// Protobuf has a 64MB limit built in, this variable will override this. Please +// do not enable this unless you fully understand the implications: protobufs +// must all be kept in memory at the same time, so if they grow too big you may +// get OOM errors. The protobuf APIs do not provide any tools for processing +// protobufs in chunks. If you have protos this big you should break them up if +// it is at all convenient to do so. +static bool allow_oversize_protos = false; + +// Provide a method in the module to set allow_oversize_protos to a boolean +// value. This method returns the newly value of allow_oversize_protos. +static PyObject* SetAllowOversizeProtos(PyObject* m, PyObject* arg) { + if (!arg || !PyBool_Check(arg)) { + PyErr_SetString(PyExc_TypeError, + "Argument to SetAllowOversizeProtos must be boolean"); + return NULL; + } + allow_oversize_protos = PyObject_IsTrue(arg); + if (allow_oversize_protos) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } +} + static PyObject* MergeFromString(CMessage* self, PyObject* arg) { const void* data; Py_ssize_t data_length; @@ -1921,15 +1945,9 @@ static PyObject* MergeFromString(CMessage* self, PyObject* arg) { AssureWritable(self); io::CodedInputStream input( reinterpret_cast<const uint8*>(data), data_length); -#if PROTOBUF_PYTHON_ALLOW_OVERSIZE_PROTOS - // Protobuf has a 64MB limit built in, this code will override this. Please do - // not enable this unless you fully understand the implications: protobufs - // must all be kept in memory at the same time, so if they grow too big you - // may get OOM errors. The protobuf APIs do not provide any tools for - // processing protobufs in chunks. If you have protos this big you should - // break them up if it is at all convenient to do so. - input.SetTotalBytesLimit(INT_MAX, INT_MAX); -#endif // PROTOBUF_PYTHON_ALLOW_OVERSIZE_PROTOS + if (allow_oversize_protos) { + input.SetTotalBytesLimit(INT_MAX, INT_MAX); + } PyDescriptorPool* pool = GetDescriptorPoolForMessage(self); input.SetExtensionRegistry(pool->pool, pool->message_factory); bool success = self->message->MergePartialFromCodedStream(&input); @@ -3046,6 +3064,11 @@ bool InitProto2MessageModule(PyObject *m) { } // namespace python } // namespace protobuf +static PyMethodDef ModuleMethods[] = { + {"SetAllowOversizeProtos", + (PyCFunction)google::protobuf::python::cmessage::SetAllowOversizeProtos, + METH_O, "Enable/disable oversize proto parsing."}, +}; #if PY_MAJOR_VERSION >= 3 static struct PyModuleDef _module = { @@ -3053,7 +3076,7 @@ static struct PyModuleDef _module = { "_message", google::protobuf::python::module_docstring, -1, - NULL, + ModuleMethods, /* m_methods */ NULL, NULL, NULL, @@ -3072,7 +3095,8 @@ extern "C" { #if PY_MAJOR_VERSION >= 3 m = PyModule_Create(&_module); #else - m = Py_InitModule3("_message", NULL, google::protobuf::python::module_docstring); + m = Py_InitModule3("_message", ModuleMethods, + google::protobuf::python::module_docstring); #endif if (m == NULL) { return INITFUNC_ERRORVAL; diff --git a/src/google/protobuf/stubs/int128.cc b/src/google/protobuf/stubs/int128.cc index d80c64f2..3a36b4b1 100644 --- a/src/google/protobuf/stubs/int128.cc +++ b/src/google/protobuf/stubs/int128.cc @@ -31,7 +31,7 @@ #include <google/protobuf/stubs/int128.h> #include <iomanip> -#include <iostream> // NOLINT(readability/streams) +#include <ostream> // NOLINT(readability/streams) #include <sstream> namespace google { diff --git a/src/google/protobuf/stubs/statusor.h b/src/google/protobuf/stubs/statusor.h index ad848701..29f869ad 100644 --- a/src/google/protobuf/stubs/statusor.h +++ b/src/google/protobuf/stubs/statusor.h @@ -224,7 +224,7 @@ inline StatusOr<T>& StatusOr<T>::operator=(const StatusOr<T>& other) { template<typename T> template<typename U> inline StatusOr<T>::StatusOr(const StatusOr<U>& other) - : status_(other.status_), value_(other.status_.ok() ? other.value_ : NULL) { + : status_(other.status_), value_(other.status_.ok() ? other.value_ : T()) { } template<typename T> @@ -1,17 +1,24 @@ -#!/usr/bin/env bash +#!/bin/bash +# +# Build and runs tests for the protobuf project. The tests as written here are +# used by both Jenkins and Travis, though some specialized logic is required to +# handle the differences between them. -# Note: travis currently does not support testing more than one language so the -# .travis.yml cheats and claims to only be cpp. If they add multiple language -# support, this should probably get updated to install steps and/or -# rvm/gemfile/jdk/etc. entries rather than manually doing the work. - -# .travis.yml uses matrix.exclude to block the cases where app-get can't be -# use to install things. +on_travis() { + if [ "$TRAVIS" == "true" ]; then + "$@" + fi +} # For when some other test needs the C++ main build, including protoc and # libprotobuf. internal_build_cpp() { - if [ $(uname -s) == "Linux" ]; then + if [ -f src/protoc ]; then + # Already built. + return + fi + + if [[ $(uname -s) == "Linux" && "$TRAVIS" == "true" ]]; then # Install GCC 4.8 to replace the default GCC 4.6. We need 4.8 for more # decent C++ 11 support in order to compile conformance tests. sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y @@ -42,16 +49,20 @@ build_csharp() { # need to really build protoc, but it's simplest to keep with the # conventions of the other builds. internal_build_cpp + NUGET=/usr/local/bin/nuget.exe - # Install latest version of Mono - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF - echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list - echo "deb http://download.mono-project.com/repo/debian wheezy-libtiff-compat main" | sudo tee -a /etc/apt/sources.list.d/mono-xamarin.list - sudo apt-get update -qq - sudo apt-get install -qq mono-devel referenceassemblies-pcl nunit - wget www.nuget.org/NuGet.exe -O nuget.exe + if [ "$TRAVIS" == "true" ]; then + # Install latest version of Mono + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF + echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list + echo "deb http://download.mono-project.com/repo/debian wheezy-libtiff-compat main" | sudo tee -a /etc/apt/sources.list.d/mono-xamarin.list + sudo apt-get update -qq + sudo apt-get install -qq mono-devel referenceassemblies-pcl nunit + wget www.nuget.org/NuGet.exe -O nuget.exe + NUGET=../../nuget.exe + fi - (cd csharp/src; mono ../../nuget.exe restore) + (cd csharp/src; mono $NUGET restore) csharp/buildall.sh cd conformance && make test_csharp && cd .. } @@ -78,40 +89,54 @@ use_java() { version=$1 case "$version" in jdk6) - sudo apt-get install openjdk-6-jdk + on_travis sudo apt-get install openjdk-6-jdk export PATH=/usr/lib/jvm/java-6-openjdk-amd64/bin:$PATH ;; jdk7) - sudo apt-get install openjdk-7-jdk + on_travis sudo apt-get install openjdk-7-jdk export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH ;; oracle7) - sudo apt-get install python-software-properties # for apt-add-repository - echo "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" | \ - sudo debconf-set-selections - yes | sudo apt-add-repository ppa:webupd8team/java - yes | sudo apt-get install oracle-java7-installer + if [ "$TRAVIS" == "true" ]; then + sudo apt-get install python-software-properties # for apt-add-repository + echo "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" | \ + sudo debconf-set-selections + yes | sudo apt-add-repository ppa:webupd8team/java + yes | sudo apt-get install oracle-java7-installer + fi; export PATH=/usr/lib/jvm/java-7-oracle/bin:$PATH ;; esac + if [ "$TRAVIS" != "true" ]; then + MAVEN_LOCAL_REPOSITORY=/var/maven_local_repository + MVN="$MVN -e -X --offline -Dmaven.repo.local=$MAVEN_LOCAL_REPOSITORY" + fi; + which java java -version } +# --batch-mode supresses download progress output that spams the logs. +MVN="mvn --batch-mode" + build_java() { + version=$1 + dir=java_$version # Java build needs `protoc`. internal_build_cpp - cd java && mvn test && mvn install - cd util && mvn test + cp -r java $dir + cd $dir && $MVN clean && $MVN test cd ../.. } +# The conformance tests are hard-coded to work with the $ROOT/java directory. +# So this can't run in parallel with two different sets of tests. build_java_with_conformance_tests() { # Java build needs `protoc`. internal_build_cpp - cd java && mvn test && mvn install - cd util && mvn test && mvn assembly:single + cd java && $MVN test && $MVN install + cd util && $MVN package assembly:single cd ../.. cd conformance && make test_java && cd .. } @@ -119,12 +144,12 @@ build_java_with_conformance_tests() { build_javanano() { # Java build needs `protoc`. internal_build_cpp - cd javanano && mvn test && cd .. + cd javanano && $MVN test && cd .. } build_java_jdk6() { use_java jdk6 - build_java + build_java jdk6 } build_java_jdk7() { use_java jdk7 @@ -132,7 +157,7 @@ build_java_jdk7() { } build_java_oracle7() { use_java oracle7 - build_java + build_java oracle7 } build_javanano_jdk6() { @@ -149,6 +174,9 @@ build_javanano_oracle7() { } internal_install_python_deps() { + if [ "$TRAVIS" != "true" ]; then + return; + fi # Install tox (OS X doesn't have pip). if [ $(uname -s) == "Darwin" ]; then sudo easy_install tox @@ -278,6 +306,14 @@ build_javascript() { cd js && npm install && npm test && cd .. } +# Note: travis currently does not support testing more than one language so the +# .travis.yml cheats and claims to only be cpp. If they add multiple language +# support, this should probably get updated to install steps and/or +# rvm/gemfile/jdk/etc. entries rather than manually doing the work. + +# .travis.yml uses matrix.exclude to block the cases where app-get can't be +# use to install things. + # -------- main -------- if [ "$#" -ne 1 ]; then @@ -294,10 +330,10 @@ Usage: $0 { cpp | objectivec_osx | python | python_cpp | - ruby_19 | - ruby_20 | - ruby_21 | - ruby_22 | + ruby19 | + ruby20 | + ruby21 | + ruby22 | jruby } " exit 1 |