diff options
20 files changed, 328 insertions, 423 deletions
diff --git a/INSTALL.txt b/INSTALL.txt deleted file mode 100644 index ce3b0949..00000000 --- a/INSTALL.txt +++ /dev/null @@ -1,237 +0,0 @@ -This file contains detailed but generic information on building and -installing the C++ part of this project. For shorter instructions, -as well as instructions for compiling and installing the Java or -Python parts, see README. - -====================================================================== - -Copyright 1994, 1995, 1996, 1999, 2000, 2001, 2002 Free Software -Foundation, Inc. - - This file is free documentation; the Free Software Foundation gives -unlimited permission to copy, distribute and modify it. - - -Basic Installation -================== - - These are generic installation instructions. - - The `configure' shell script attempts to guess correct values for -various system-dependent variables used during compilation. It uses -those values to create a `Makefile' in each directory of the package. -It may also create one or more `.h' files containing system-dependent -definitions. Finally, it creates a shell script `config.status' that -you can run in the future to recreate the current configuration, and a -file `config.log' containing compiler output (useful mainly for -debugging `configure'). - - It can also use an optional file (typically called `config.cache' -and enabled with `--cache-file=config.cache' or simply `-C') that saves -the results of its tests to speed up reconfiguring. (Caching is -disabled by default to prevent problems with accidental use of stale -cache files.) - - If you need to do unusual things to compile the package, please try -to figure out how `configure' could check whether to do them, and mail -diffs or instructions to the address given in the `README' so they can -be considered for the next release. If you are using the cache, and at -some point `config.cache' contains results you don't want to keep, you -may remove or edit it. - - The file `configure.ac' (or `configure.in') is used to create -`configure' by a program called `autoconf'. You only need -`configure.ac' if you want to change it or regenerate `configure' using -a newer version of `autoconf'. - -The simplest way to compile this package is: - - 1. `cd' to the directory containing the package's source code and type - `./configure' to configure the package for your system. If you're - using `csh' on an old version of System V, you might need to type - `sh ./configure' instead to prevent `csh' from trying to execute - `configure' itself. - - Running `configure' takes awhile. While running, it prints some - messages telling which features it is checking for. - - 2. Type `make' to compile the package. - - 3. Optionally, type `make check' to run any self-tests that come with - the package. - - 4. Type `make install' to install the programs and any data files and - documentation. - - 5. You can remove the program binaries and object files from the - source code directory by typing `make clean'. To also remove the - files that `configure' created (so you can compile the package for - a different kind of computer), type `make distclean'. There is - also a `make maintainer-clean' target, but that is intended mainly - for the package's developers. If you use it, you may have to get - all sorts of other programs in order to regenerate files that came - with the distribution. - -Compilers and Options -===================== - - Some systems require unusual options for compilation or linking that -the `configure' script does not know about. Run `./configure --help' -for details on some of the pertinent environment variables. - - You can give `configure' initial values for configuration parameters -by setting variables in the command line or in the environment. Here -is an example: - - ./configure CC=c89 CFLAGS=-O2 LIBS=-lposix - - *Note Defining Variables::, for more details. - -Compiling For Multiple Architectures -==================================== - - You can compile the package for more than one kind of computer at the -same time, by placing the object files for each architecture in their -own directory. To do this, you must use a version of `make' that -supports the `VPATH' variable, such as GNU `make'. `cd' to the -directory where you want the object files and executables to go and run -the `configure' script. `configure' automatically checks for the -source code in the directory that `configure' is in and in `..'. - - If you have to use a `make' that does not support the `VPATH' -variable, you have to compile the package for one architecture at a -time in the source code directory. After you have installed the -package for one architecture, use `make distclean' before reconfiguring -for another architecture. - -Installation Names -================== - - By default, `make install' will install the package's files in -`/usr/local/bin', `/usr/local/man', etc. You can specify an -installation prefix other than `/usr/local' by giving `configure' the -option `--prefix=PATH'. - - You can specify separate installation prefixes for -architecture-specific files and architecture-independent files. If you -give `configure' the option `--exec-prefix=PATH', the package will use -PATH as the prefix for installing programs and libraries. -Documentation and other data files will still use the regular prefix. - - In addition, if you use an unusual directory layout you can give -options like `--bindir=PATH' to specify different values for particular -kinds of files. Run `configure --help' for a list of the directories -you can set and what kinds of files go in them. - - If the package supports it, you can cause programs to be installed -with an extra prefix or suffix on their names by giving `configure' the -option `--program-prefix=PREFIX' or `--program-suffix=SUFFIX'. - -Optional Features -================= - - Some packages pay attention to `--enable-FEATURE' options to -`configure', where FEATURE indicates an optional part of the package. -They may also pay attention to `--with-PACKAGE' options, where PACKAGE -is something like `gnu-as' or `x' (for the X Window System). The -`README' should mention any `--enable-' and `--with-' options that the -package recognizes. - - For packages that use the X Window System, `configure' can usually -find the X include and library files automatically, but if it doesn't, -you can use the `configure' options `--x-includes=DIR' and -`--x-libraries=DIR' to specify their locations. - -Specifying the System Type -========================== - - There may be some features `configure' cannot figure out -automatically, but needs to determine by the type of machine the package -will run on. Usually, assuming the package is built to be run on the -_same_ architectures, `configure' can figure that out, but if it prints -a message saying it cannot guess the machine type, give it the -`--build=TYPE' option. TYPE can either be a short name for the system -type, such as `sun4', or a canonical name which has the form: - - CPU-COMPANY-SYSTEM - -where SYSTEM can have one of these forms: - - OS KERNEL-OS - - See the file `config.sub' for the possible values of each field. If -`config.sub' isn't included in this package, then this package doesn't -need to know the machine type. - - If you are _building_ compiler tools for cross-compiling, you should -use the `--target=TYPE' option to select the type of system they will -produce code for. - - If you want to _use_ a cross compiler, that generates code for a -platform different from the build platform, you should specify the -"host" platform (i.e., that on which the generated programs will -eventually be run) with `--host=TYPE'. - -Sharing Defaults -================ - - If you want to set default values for `configure' scripts to share, -you can create a site shell script called `config.site' that gives -default values for variables like `CC', `cache_file', and `prefix'. -`configure' looks for `PREFIX/share/config.site' if it exists, then -`PREFIX/etc/config.site' if it exists. Or, you can set the -`CONFIG_SITE' environment variable to the location of the site script. -A warning: not all `configure' scripts look for a site script. - -Defining Variables -================== - - Variables not defined in a site shell script can be set in the -environment passed to `configure'. However, some packages may run -configure again during the build, and the customized values of these -variables may be lost. In order to avoid this problem, you should set -them in the `configure' command line, using `VAR=value'. For example: - - ./configure CC=/usr/local2/bin/gcc - -will cause the specified gcc to be used as the C compiler (unless it is -overridden in the site shell script). - -`configure' Invocation -====================== - - `configure' recognizes the following options to control how it -operates. - -`--help' -`-h' - Print a summary of the options to `configure', and exit. - -`--version' -`-V' - Print the version of Autoconf used to generate the `configure' - script, and exit. - -`--cache-file=FILE' - Enable the cache: use and save the results of the tests in FILE, - traditionally `config.cache'. FILE defaults to `/dev/null' to - disable caching. - -`--config-cache' -`-C' - Alias for `--cache-file=config.cache'. - -`--quiet' -`--silent' -`-q' - Do not print messages saying which checks are being made. To - suppress all normal output, redirect it to `/dev/null' (any error - messages will still be shown). - -`--srcdir=DIR' - Look for the package's source code in directory DIR. Usually - `configure' can determine that directory automatically. - -`configure' also accepts some other, not widely useful, options. Run -`configure --help' for more details. - diff --git a/Makefile.am b/Makefile.am index 49ad640c..3d39a898 100644 --- a/Makefile.am +++ b/Makefile.am @@ -697,7 +697,6 @@ EXTRA_DIST = $(@DIST_LANG@_EXTRA_DIST) \ autogen.sh \ generate_descriptor_proto.sh \ README.md \ - INSTALL.txt \ LICENSE \ CONTRIBUTORS.txt \ CHANGES.txt \ @@ -10,6 +10,20 @@ https://developers.google.com/protocol-buffers/ C++ Installation - Unix ----------------------- +To build protobuf from source, the following tools are needed: + + * autoconf + * automake + * libtool + * curl (used to download gmock) + +On Ubuntu, you can install them with: + + $ sudo apt-get install autoconf automake libtool curl + +On other platforms, please use the corresponding package managing tool to +install them before proceeding. + If you get the source from github, you need to generate the configure script first: @@ -28,15 +42,17 @@ Buffer compiler (protoc) execute the following: $ ./configure $ make $ make check - $ make install + $ sudo make install + $ sudo ldconfig # refresh shared library cache. If "make check" fails, you can still install, but it is likely that some features of this library will not work correctly on your system. Proceed at your own risk. -"make install" may require superuser privileges. +For advanced usage information on configure and make, please refer to the +autoconf documentation: -For advanced usage information on configure and make, see INSTALL.txt. + http://www.gnu.org/software/autoconf/manual/autoconf.html#Running-configure-Scripts **Hint on install location** @@ -156,9 +172,18 @@ For advanced usage information on configure and make, see INSTALL.txt. C++ Installation - Windows -------------------------- -If you are using Microsoft Visual C++, see cmake/README.md. +If you only need the protoc binary, you can download it from the release +page: + + https://github.com/google/protobuf/releases + +In the downloads section, download the zip file protoc-$VERSION-win32.zip. +It contains the protoc binary as well as public proto files of protobuf +library. + +To build from source using Microsoft Visual C++, see cmake/README.md. -If you are using Cygwin or MinGW, follow the Unix installation +To build from source using Cygwin or MinGW, follow the Unix installation instructions, above. Binary Compatibility Warning diff --git a/conformance/conformance_objc.m b/conformance/conformance_objc.m index 06c97a82..1124bfeb 100644 --- a/conformance/conformance_objc.m +++ b/conformance/conformance_objc.m @@ -95,7 +95,7 @@ static ConformanceResponse *DoTest(ConformanceRequest *request) { case WireFormat_Protobuf: response.protobufPayload = testMessage.data; if (!response.protobufPayload) { - response.runtimeError = + response.serializeError = [NSString stringWithFormat:@"Failed to make data from: %@", testMessage]; } break; diff --git a/conformance/conformance_test.cc b/conformance/conformance_test.cc index c39bf26d..a232ecc5 100644 --- a/conformance/conformance_test.cc +++ b/conformance/conformance_test.cc @@ -356,6 +356,8 @@ void ConformanceTestSuite::ExpectParseFailureForProto( RunTest(effective_test_name, request, &response); if (response.result_case() == ConformanceResponse::kParseError) { ReportSuccess(effective_test_name); + } else if (response.result_case() == ConformanceResponse::kSkipped) { + ReportSkip(effective_test_name, request, response); } else { ReportFailure(effective_test_name, request, response, "Should have failed to parse, but didn't."); @@ -408,6 +410,11 @@ void ConformanceTestSuite::RunValidJsonTestWithValidator( RunTest(effective_test_name, request, &response); + if (response.result_case() == ConformanceResponse::kSkipped) { + ReportSkip(effective_test_name, request, response); + return; + } + if (response.result_case() != ConformanceResponse::kJsonPayload) { ReportFailure(effective_test_name, request, response, "Expected JSON payload but got type %d.", @@ -444,6 +451,8 @@ void ConformanceTestSuite::ExpectParseFailureForJson( RunTest(effective_test_name, request, &response); if (response.result_case() == ConformanceResponse::kParseError) { ReportSuccess(effective_test_name); + } else if (response.result_case() == ConformanceResponse::kSkipped) { + ReportSkip(effective_test_name, request, response); } else { ReportFailure(effective_test_name, request, response, "Should have failed to parse, but didn't."); @@ -466,6 +475,8 @@ void ConformanceTestSuite::ExpectSerializeFailureForJson( RunTest(effective_test_name, request, &response); if (response.result_case() == ConformanceResponse::kSerializeError) { ReportSuccess(effective_test_name); + } else if (response.result_case() == ConformanceResponse::kSkipped) { + ReportSkip(effective_test_name, request, response); } else { ReportFailure(effective_test_name, request, response, "Should have failed to serialize, but didn't."); diff --git a/conformance/failure_list_objc.txt b/conformance/failure_list_objc.txt index b13de084..5dac3501 100644 --- a/conformance/failure_list_objc.txt +++ b/conformance/failure_list_objc.txt @@ -1,107 +1,4 @@ -# TODO(xiaofeng): These tests should be skipped (because objc doesn't support -# JSON) but currently they are reported as failures by the objc testee. -DurationProtoInputTooLarge.JsonOutput -DurationProtoInputTooSmall.JsonOutput -FieldMaskNumbersDontRoundTrip.JsonOutput -FieldMaskPathsDontRoundTrip.JsonOutput -FieldMaskTooManyUnderscore.JsonOutput -JsonInput.BoolFieldAllCapitalFalse -JsonInput.BoolFieldAllCapitalTrue -JsonInput.BoolFieldCamelCaseFalse -JsonInput.BoolFieldCamelCaseTrue -JsonInput.BoolFieldDoubleQuotedFalse -JsonInput.BoolFieldDoubleQuotedTrue -JsonInput.BoolFieldIntegerOne -JsonInput.BoolFieldIntegerZero -JsonInput.BoolMapFieldKeyNotQuoted -JsonInput.BytesFieldInvalidBase64Characters -JsonInput.BytesFieldNoPadding -JsonInput.DoubleFieldInfinityNotQuoted -JsonInput.DoubleFieldNanNotQuoted -JsonInput.DoubleFieldNegativeInfinityNotQuoted -JsonInput.DoubleFieldTooLarge -JsonInput.DoubleFieldTooSmall -JsonInput.DurationHas3FractionalDigits.Validator -JsonInput.DurationHas6FractionalDigits.Validator -JsonInput.DurationHas9FractionalDigits.Validator -JsonInput.DurationHasZeroFractionalDigit.Validator -JsonInput.DurationJsonInputTooLarge -JsonInput.DurationJsonInputTooSmall -JsonInput.DurationMissingS -JsonInput.EnumFieldNotQuoted -JsonInput.EnumFieldUnknownValue.Validator -JsonInput.FieldMaskInvalidCharacter -JsonInput.FieldNameDuplicate -JsonInput.FieldNameDuplicateDifferentCasing1 -JsonInput.FieldNameDuplicateDifferentCasing2 -JsonInput.FieldNameInLowerCamelCase.Validator -JsonInput.FieldNameNotQuoted -JsonInput.FieldNameWithMixedCases.Validator -JsonInput.FieldNameWithNumbers.Validator -JsonInput.FloatFieldInfinityNotQuoted -JsonInput.FloatFieldNanNotQuoted -JsonInput.FloatFieldNegativeInfinityNotQuoted -JsonInput.FloatFieldTooLarge -JsonInput.FloatFieldTooSmall -JsonInput.Int32FieldLeadingSpace -JsonInput.Int32FieldLeadingZero -JsonInput.Int32FieldNegativeWithLeadingZero -JsonInput.Int32FieldNotInteger -JsonInput.Int32FieldNotNumber -JsonInput.Int32FieldPlusSign -JsonInput.Int32FieldTooLarge -JsonInput.Int32FieldTooSmall -JsonInput.Int32FieldTrailingSpace -JsonInput.Int32MapFieldKeyNotQuoted -JsonInput.Int64FieldBeString.Validator -JsonInput.Int64FieldNotInteger -JsonInput.Int64FieldNotNumber -JsonInput.Int64FieldTooLarge -JsonInput.Int64FieldTooSmall -JsonInput.Int64MapFieldKeyNotQuoted -JsonInput.JsonWithComments -JsonInput.MapFieldKeyIsNull -JsonInput.MapFieldValueIsNull -JsonInput.OneofFieldDuplicate -JsonInput.RepeatedFieldMessageElementIsNull -JsonInput.RepeatedFieldPrimitiveElementIsNull -JsonInput.RepeatedFieldTrailingComma -JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool -JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotMessage -JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotString -JsonInput.RepeatedFieldWrongElementTypeExpectingMessagesGotBool -JsonInput.RepeatedFieldWrongElementTypeExpectingMessagesGotInt -JsonInput.RepeatedFieldWrongElementTypeExpectingMessagesGotString -JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool -JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt -JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotMessage -JsonInput.StringFieldInvalidEscape -JsonInput.StringFieldNotAString -JsonInput.StringFieldSurrogateInWrongOrder -JsonInput.StringFieldUnpairedHighSurrogate -JsonInput.StringFieldUnpairedLowSurrogate -JsonInput.StringFieldUnterminatedEscape -JsonInput.StringFieldUppercaseEscapeLetter -JsonInput.TimestampHas3FractionalDigits.Validator -JsonInput.TimestampHas6FractionalDigits.Validator -JsonInput.TimestampHas9FractionalDigits.Validator -JsonInput.TimestampHasZeroFractionalDigit.Validator -JsonInput.TimestampJsonInputLowercaseT -JsonInput.TimestampJsonInputLowercaseZ -JsonInput.TimestampJsonInputMissingT -JsonInput.TimestampJsonInputMissingZ -JsonInput.TimestampJsonInputTooLarge -JsonInput.TimestampJsonInputTooSmall -JsonInput.TimestampZeroNormalized.Validator -JsonInput.TrailingCommaInAnObject -JsonInput.Uint32FieldNotInteger -JsonInput.Uint32FieldNotNumber -JsonInput.Uint32FieldTooLarge -JsonInput.Uint32MapFieldKeyNotQuoted -JsonInput.Uint64FieldBeString.Validator -JsonInput.Uint64FieldNotInteger -JsonInput.Uint64FieldNotNumber -JsonInput.Uint64FieldTooLarge -JsonInput.Uint64MapFieldKeyNotQuoted -TimestampProtoInputTooLarge.JsonOutput -TimestampProtoInputTooSmall.JsonOutput +# No tests currently failing. +# +# json input or output tests are skipped (in conformance_objc.m) as mobile +# platforms don't support json wire format to avoid code bloat. diff --git a/csharp/src/Google.Protobuf.Test/JsonParserTest.cs b/csharp/src/Google.Protobuf.Test/JsonParserTest.cs index 3babb391..303baacc 100644 --- a/csharp/src/Google.Protobuf.Test/JsonParserTest.cs +++ b/csharp/src/Google.Protobuf.Test/JsonParserTest.cs @@ -39,9 +39,7 @@ using System; namespace Google.Protobuf { /// <summary> - /// Unit tests for JSON parsing. Some tests are ignored at the moment as the desired behaviour - /// isn't fully known, either in terms of which exceptions should be thrown or whether they should - /// count as valid values. + /// Unit tests for JSON parsing. /// </summary> public class JsonParserTest { @@ -205,6 +203,8 @@ namespace Google.Protobuf [Test] [TestCase("+0")] + [TestCase(" 1")] + [TestCase("1 ")] [TestCase("00")] [TestCase("-00")] [TestCase("--1")] @@ -318,7 +318,18 @@ namespace Google.Protobuf [TestCase("1.0.0")] [TestCase("+1")] [TestCase("00")] + [TestCase("01")] + [TestCase("-00")] + [TestCase("-01")] [TestCase("--1")] + [TestCase(" Infinity")] + [TestCase(" -Infinity")] + [TestCase("NaN ")] + [TestCase("Infinity ")] + [TestCase("-Infinity ")] + [TestCase(" NaN")] + [TestCase("INFINITY")] + [TestCase("nan")] [TestCase("\u00BD")] // 1/2 as a single Unicode character. Just sanity checking... public void StringToDouble_Invalid(string jsonValue) { @@ -363,6 +374,10 @@ namespace Google.Protobuf [TestCase("-1", -1)] [TestCase("2147483647", 2147483647)] [TestCase("-2147483648", -2147483648)] + [TestCase("1e1", 10)] + [TestCase("-1e1", -10)] + [TestCase("10.00", 10)] + [TestCase("-10.00", -10)] public void NumberToInt32_Valid(string jsonValue, int expectedParsedValue) { string json = "{ \"singleInt32\": " + jsonValue + "}"; @@ -376,7 +391,8 @@ namespace Google.Protobuf [TestCase("-00", typeof(InvalidJsonException))] [TestCase("--1", typeof(InvalidJsonException))] [TestCase("+1", typeof(InvalidJsonException))] - [TestCase("1.5", typeof(InvalidProtocolBufferException), Ignore = true, Reason = "Desired behaviour unclear")] + [TestCase("1.5", typeof(InvalidProtocolBufferException))] + // Value is out of range [TestCase("1e10", typeof(InvalidProtocolBufferException))] [TestCase("2147483648", typeof(InvalidProtocolBufferException))] [TestCase("-2147483649", typeof(InvalidProtocolBufferException))] @@ -411,8 +427,10 @@ namespace Google.Protobuf [TestCase("0", 0L)] [TestCase("1", 1L)] [TestCase("-1", -1L)] - [TestCase("9223372036854775807", 9223372036854775807, Ignore = true, Reason = "Desired behaviour unclear")] - [TestCase("-9223372036854775808", -9223372036854775808, Ignore = true, Reason = "Desired behaviour unclear")] + // long.MaxValue isn't actually representable as a double. This string value is the highest + // representable value which isn't greater than long.MaxValue. + [TestCase("9223372036854769664", 9223372036854769664)] + [TestCase("-9223372036854775808", -9223372036854775808)] public void NumberToInt64_Valid(string jsonValue, long expectedParsedValue) { string json = "{ \"singleInt64\": " + jsonValue + "}"; @@ -422,8 +440,11 @@ namespace Google.Protobuf // Assume that anything non-bounds-related is covered in the Int32 case [Test] - [TestCase("-9223372036854775809", Ignore = true, Reason = "Desired behaviour unclear")] - [TestCase("9223372036854775808", Ignore = true, Reason = "Desired behaviour unclear")] + [TestCase("9223372036854775808")] + // Theoretical bound would be -9223372036854775809, but when that is parsed to a double + // we end up with the exact value of long.MinValue due to lack of precision. The value here + // is the "next double down". + [TestCase("-9223372036854780000")] public void NumberToInt64_Invalid(string jsonValue) { string json = "{ \"singleInt64\": " + jsonValue + "}"; @@ -433,7 +454,9 @@ namespace Google.Protobuf [Test] [TestCase("0", 0UL)] [TestCase("1", 1UL)] - [TestCase("18446744073709551615", 18446744073709551615, Ignore = true, Reason = "Desired behaviour unclear")] + // ulong.MaxValue isn't representable as a double. This value is the largest double within + // the range of ulong. + [TestCase("18446744073709500416", 18446744073709500416UL)] public void NumberToUInt64_Valid(string jsonValue, ulong expectedParsedValue) { string json = "{ \"singleUint64\": " + jsonValue + "}"; @@ -475,9 +498,9 @@ namespace Google.Protobuf } [Test] - [TestCase("1.7977e308", Ignore = true, Reason = "Desired behaviour unclear")] - [TestCase("-1.7977e308", Ignore = true, Reason = "Desired behaviour unclear")] - [TestCase("1e309", Ignore = true, Reason = "Desired behaviour unclear")] + [TestCase("1.7977e308")] + [TestCase("-1.7977e308")] + [TestCase("1e309")] [TestCase("1,0")] [TestCase("1.0.0")] [TestCase("+1")] diff --git a/csharp/src/Google.Protobuf/FieldCodec.cs b/csharp/src/Google.Protobuf/FieldCodec.cs index 6ee8e2f9..c2b8d268 100644 --- a/csharp/src/Google.Protobuf/FieldCodec.cs +++ b/csharp/src/Google.Protobuf/FieldCodec.cs @@ -354,11 +354,6 @@ namespace Google.Protobuf // Otherwise it's the default value of the CLR type } - private static Func<T, bool> CreateDefaultValueCheck<TTmp>(Func<TTmp, bool> check) - { - return (Func<T, bool>)(object)check; - } - private readonly Func<CodedInputStream, T> reader; private readonly Action<CodedOutputStream, T> writer; private readonly Func<T, int> sizeCalculator; diff --git a/csharp/src/Google.Protobuf/JsonParser.cs b/csharp/src/Google.Protobuf/JsonParser.cs index 95f9ad35..300a66b4 100644 --- a/csharp/src/Google.Protobuf/JsonParser.cs +++ b/csharp/src/Google.Protobuf/JsonParser.cs @@ -540,17 +540,17 @@ namespace Google.Protobuf case FieldType.Int32: case FieldType.SInt32: case FieldType.SFixed32: - return ParseNumericString(keyText, int.Parse, false); + return ParseNumericString(keyText, int.Parse); case FieldType.UInt32: case FieldType.Fixed32: - return ParseNumericString(keyText, uint.Parse, false); + return ParseNumericString(keyText, uint.Parse); case FieldType.Int64: case FieldType.SInt64: case FieldType.SFixed64: - return ParseNumericString(keyText, long.Parse, false); + return ParseNumericString(keyText, long.Parse); case FieldType.UInt64: case FieldType.Fixed64: - return ParseNumericString(keyText, ulong.Parse, false); + return ParseNumericString(keyText, ulong.Parse); default: throw new InvalidProtocolBufferException("Invalid field type for map: " + field.FieldType); } @@ -561,7 +561,6 @@ namespace Google.Protobuf double value = token.NumberValue; checked { - // TODO: Validate that it's actually an integer, possibly in terms of the textual representation? try { switch (field.FieldType) @@ -569,16 +568,20 @@ namespace Google.Protobuf case FieldType.Int32: case FieldType.SInt32: case FieldType.SFixed32: + CheckInteger(value); return (int) value; case FieldType.UInt32: case FieldType.Fixed32: + CheckInteger(value); return (uint) value; case FieldType.Int64: case FieldType.SInt64: case FieldType.SFixed64: + CheckInteger(value); return (long) value; case FieldType.UInt64: case FieldType.Fixed64: + CheckInteger(value); return (ulong) value; case FieldType.Double: return value; @@ -597,20 +600,32 @@ namespace Google.Protobuf { return float.NegativeInfinity; } - throw new InvalidProtocolBufferException("Value out of range: " + value); + throw new InvalidProtocolBufferException($"Value out of range: {value}"); } return (float) value; default: - throw new InvalidProtocolBufferException("Unsupported conversion from JSON number for field type " + field.FieldType); + throw new InvalidProtocolBufferException($"Unsupported conversion from JSON number for field type {field.FieldType}"); } } catch (OverflowException) { - throw new InvalidProtocolBufferException("Value out of range: " + value); + throw new InvalidProtocolBufferException($"Value out of range: {value}"); } } } + private static void CheckInteger(double value) + { + if (double.IsInfinity(value) || double.IsNaN(value)) + { + throw new InvalidProtocolBufferException($"Value not an integer: {value}"); + } + if (value != Math.Floor(value)) + { + throw new InvalidProtocolBufferException($"Value not an integer: {value}"); + } + } + private static object ParseSingleStringValue(FieldDescriptor field, string text) { switch (field.FieldType) @@ -622,43 +637,35 @@ namespace Google.Protobuf case FieldType.Int32: case FieldType.SInt32: case FieldType.SFixed32: - return ParseNumericString(text, int.Parse, false); + return ParseNumericString(text, int.Parse); case FieldType.UInt32: case FieldType.Fixed32: - return ParseNumericString(text, uint.Parse, false); + return ParseNumericString(text, uint.Parse); case FieldType.Int64: case FieldType.SInt64: case FieldType.SFixed64: - return ParseNumericString(text, long.Parse, false); + return ParseNumericString(text, long.Parse); case FieldType.UInt64: case FieldType.Fixed64: - return ParseNumericString(text, ulong.Parse, false); + return ParseNumericString(text, ulong.Parse); case FieldType.Double: - double d = ParseNumericString(text, double.Parse, true); - // double.Parse can return +/- infinity on Mono for non-infinite values which are out of range for double. - if (double.IsInfinity(d) && !text.Contains("Infinity")) - { - throw new InvalidProtocolBufferException("Invalid numeric value: " + text); - } + double d = ParseNumericString(text, double.Parse); + ValidateInfinityAndNan(text, double.IsPositiveInfinity(d), double.IsNegativeInfinity(d), double.IsNaN(d)); return d; case FieldType.Float: - float f = ParseNumericString(text, float.Parse, true); - // float.Parse can return +/- infinity on Mono for non-infinite values which are out of range for float. - if (float.IsInfinity(f) && !text.Contains("Infinity")) - { - throw new InvalidProtocolBufferException("Invalid numeric value: " + text); - } + float f = ParseNumericString(text, float.Parse); + ValidateInfinityAndNan(text, float.IsPositiveInfinity(f), float.IsNegativeInfinity(f), float.IsNaN(f)); return f; case FieldType.Enum: var enumValue = field.EnumType.FindValueByName(text); if (enumValue == null) { - throw new InvalidProtocolBufferException("Invalid enum value: " + text + " for enum type: " + field.EnumType.FullName); + throw new InvalidProtocolBufferException($"Invalid enum value: {text} for enum type: {field.EnumType.FullName}"); } // Just return it as an int, and let the CLR convert it. return enumValue.Number; default: - throw new InvalidProtocolBufferException("Unsupported conversion from JSON string for field type " + field.FieldType); + throw new InvalidProtocolBufferException($"Unsupported conversion from JSON string for field type {field.FieldType}"); } } @@ -670,43 +677,53 @@ namespace Google.Protobuf return field.MessageType.Parser.CreateTemplate(); } - private static T ParseNumericString<T>(string text, Func<string, NumberStyles, IFormatProvider, T> parser, bool floatingPoint) + private static T ParseNumericString<T>(string text, Func<string, NumberStyles, IFormatProvider, T> parser) { - // TODO: Prohibit leading zeroes (but allow 0!) - // TODO: Validate handling of "Infinity" etc. (Should be case sensitive, no leading whitespace etc) // Can't prohibit this with NumberStyles. if (text.StartsWith("+")) { - throw new InvalidProtocolBufferException("Invalid numeric value: " + text); + throw new InvalidProtocolBufferException($"Invalid numeric value: {text}"); } if (text.StartsWith("0") && text.Length > 1) { if (text[1] >= '0' && text[1] <= '9') { - throw new InvalidProtocolBufferException("Invalid numeric value: " + text); + throw new InvalidProtocolBufferException($"Invalid numeric value: {text}"); } } else if (text.StartsWith("-0") && text.Length > 2) { if (text[2] >= '0' && text[2] <= '9') { - throw new InvalidProtocolBufferException("Invalid numeric value: " + text); + throw new InvalidProtocolBufferException($"Invalid numeric value: {text}"); } } try { - var styles = floatingPoint - ? NumberStyles.AllowLeadingSign | NumberStyles.AllowDecimalPoint | NumberStyles.AllowExponent - : NumberStyles.AllowLeadingSign; - return parser(text, styles, CultureInfo.InvariantCulture); + return parser(text, NumberStyles.AllowLeadingSign | NumberStyles.AllowDecimalPoint | NumberStyles.AllowExponent, CultureInfo.InvariantCulture); } catch (FormatException) { - throw new InvalidProtocolBufferException("Invalid numeric value for type: " + text); + throw new InvalidProtocolBufferException($"Invalid numeric value for type: {text}"); } catch (OverflowException) { - throw new InvalidProtocolBufferException("Value out of range: " + text); + throw new InvalidProtocolBufferException($"Value out of range: {text}"); + } + } + + /// <summary> + /// Checks that any infinite/NaN values originated from the correct text. + /// This corrects the lenient whitespace handling of double.Parse/float.Parse, as well as the + /// way that Mono parses out-of-range values as infinity. + /// </summary> + private static void ValidateInfinityAndNan(string text, bool isPositiveInfinity, bool isNegativeInfinity, bool isNaN) + { + if ((isPositiveInfinity && text != "Infinity") || + (isNegativeInfinity && text != "-Infinity") || + (isNaN && text != "NaN")) + { + throw new InvalidProtocolBufferException($"Invalid numeric value: {text}"); } } @@ -719,7 +736,7 @@ namespace Google.Protobuf var match = TimestampRegex.Match(token.StringValue); if (!match.Success) { - throw new InvalidProtocolBufferException("Invalid Timestamp value: " + token.StringValue); + throw new InvalidProtocolBufferException($"Invalid Timestamp value: {token.StringValue}"); } var dateTime = match.Groups["datetime"].Value; var subseconds = match.Groups["subseconds"].Value; diff --git a/java/util/src/main/java/com/google/protobuf/util/FieldMaskUtil.java b/java/util/src/main/java/com/google/protobuf/util/FieldMaskUtil.java index 535be0fa..0b3060a7 100644 --- a/java/util/src/main/java/com/google/protobuf/util/FieldMaskUtil.java +++ b/java/util/src/main/java/com/google/protobuf/util/FieldMaskUtil.java @@ -119,7 +119,7 @@ public class FieldMaskUtil { /** * Constructs a FieldMask from the passed field numbers. * - * @throws IllegalArugmentException if any of the fields are invalid for the message. + * @throws IllegalArgumentException if any of the fields are invalid for the message. */ public static FieldMask fromFieldNumbers(Class<? extends Message> type, int... fieldNumbers) { return fromFieldNumbers(type, Ints.asList(fieldNumbers)); @@ -128,7 +128,7 @@ public class FieldMaskUtil { /** * Constructs a FieldMask from the passed field numbers. * - * @throws IllegalArugmentException if any of the fields are invalid for the message. + * @throws IllegalArgumentException if any of the fields are invalid for the message. */ public static FieldMask fromFieldNumbers( Class<? extends Message> type, Iterable<Integer> fieldNumbers) { diff --git a/javanano/pom.xml b/javanano/pom.xml index 1a95a43e..2c9dd943 100644 --- a/javanano/pom.xml +++ b/javanano/pom.xml @@ -10,7 +10,7 @@ </parent> <groupId>com.google.protobuf.nano</groupId> <artifactId>protobuf-javanano</artifactId> - <version>3.0.0-alpha-4</version> + <version>3.0.0-alpha-5</version> <packaging>bundle</packaging> <name>Protocol Buffer JavaNano API</name> <description> @@ -165,7 +165,7 @@ <instructions> <Bundle-DocURL>https://developers.google.com/protocol-buffers/</Bundle-DocURL> <Bundle-SymbolicName>com.google.protobuf</Bundle-SymbolicName> - <Export-Package>com.google.protobuf;version=3.0.0-alpha-4</Export-Package> + <Export-Package>com.google.protobuf;version=3.0.0-alpha-5</Export-Package> </instructions> </configuration> </plugin> diff --git a/javanano/src/main/java/com/google/protobuf/nano/FieldArray.java b/javanano/src/main/java/com/google/protobuf/nano/FieldArray.java index eca9c0d9..b49a97fa 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/FieldArray.java +++ b/javanano/src/main/java/com/google/protobuf/nano/FieldArray.java @@ -32,13 +32,13 @@ package com.google.protobuf.nano; /** - * A custom version of {@link android.util.SparseArray} with the minimal API + * A custom version of {@code android.util.SparseArray} with the minimal API * for storing {@link FieldData} objects. * * <p>This class is an internal implementation detail of nano and should not * be called directly by clients. * - * Based on {@link android.support.v4.util.SpareArrayCompat}. + * Based on {@code android.support.v4.util.SpareArrayCompat}. */ public final class FieldArray implements Cloneable { private static final FieldData DELETED = new FieldData(); diff --git a/objectivec/ProtocolBuffers_OSX.xcodeproj/xcshareddata/xcschemes/PerformanceTests.xcscheme b/objectivec/ProtocolBuffers_OSX.xcodeproj/xcshareddata/xcschemes/PerformanceTests.xcscheme index 7ddf283d..25814c5f 100644 --- a/objectivec/ProtocolBuffers_OSX.xcodeproj/xcshareddata/xcschemes/PerformanceTests.xcscheme +++ b/objectivec/ProtocolBuffers_OSX.xcodeproj/xcshareddata/xcschemes/PerformanceTests.xcscheme @@ -159,6 +159,9 @@ Identifier = "GPBInt64UInt64DictionaryTests"> </Test> <Test + Identifier = "GPBObjectiveCPlusPlusTests"> + </Test> + <Test Identifier = "GPBStringBoolDictionaryTests"> </Test> <Test @@ -180,9 +183,6 @@ Identifier = "GPBStringInt64DictionaryTests"> </Test> <Test - Identifier = "GPBStringTests"> - </Test> - <Test Identifier = "GPBStringUInt32DictionaryTests"> </Test> <Test diff --git a/objectivec/ProtocolBuffers_iOS.xcodeproj/xcshareddata/xcschemes/PerformanceTests.xcscheme b/objectivec/ProtocolBuffers_iOS.xcodeproj/xcshareddata/xcschemes/PerformanceTests.xcscheme index 100bd828..0b96b75f 100644 --- a/objectivec/ProtocolBuffers_iOS.xcodeproj/xcshareddata/xcschemes/PerformanceTests.xcscheme +++ b/objectivec/ProtocolBuffers_iOS.xcodeproj/xcshareddata/xcschemes/PerformanceTests.xcscheme @@ -159,6 +159,9 @@ Identifier = "GPBInt64UInt64DictionaryTests"> </Test> <Test + Identifier = "GPBObjectiveCPlusPlusTests"> + </Test> + <Test Identifier = "GPBStringBoolDictionaryTests"> </Test> <Test @@ -180,9 +183,6 @@ Identifier = "GPBStringInt64DictionaryTests"> </Test> <Test - Identifier = "GPBStringTests"> - </Test> - <Test Identifier = "GPBStringUInt32DictionaryTests"> </Test> <Test diff --git a/src/google/protobuf/compiler/cpp/cpp_message.cc b/src/google/protobuf/compiler/cpp/cpp_message.cc index af409c29..8304ebbd 100644 --- a/src/google/protobuf/compiler/cpp/cpp_message.cc +++ b/src/google/protobuf/compiler/cpp/cpp_message.cc @@ -1772,6 +1772,17 @@ GenerateShutdownCode(io::Printer* printer) { void MessageGenerator:: GenerateClassMethods(io::Printer* printer) { + // mutable_unknown_fields wrapper function for LazyStringOutputStream + // callback. + if (!UseUnknownFieldSet(descriptor_->file())) { + printer->Print( + "static ::std::string* MutableUnknownFieldsFor$classname$(\n" + " $classname$* ptr) {\n" + " return ptr->mutable_unknown_fields();\n" + "}\n" + "\n", + "classname", classname_); + } if (IsAnyMessage(descriptor_)) { printer->Print( "void $classname$::PackFrom(const ::google::protobuf::Message& message) {\n" @@ -2814,7 +2825,9 @@ GenerateMergeFrom(io::Printer* printer) { "}\n"); } else { printer->Print( - "mutable_unknown_fields()->append(from.unknown_fields());\n"); + "if (!from.unknown_fields().empty()) {\n" + " mutable_unknown_fields()->append(from.unknown_fields());\n" + "}\n"); } } @@ -2889,11 +2902,16 @@ GenerateMergeFromCodedStream(io::Printer* printer) { "classname", classname_); if (!UseUnknownFieldSet(descriptor_->file())) { + // Use LazyStringOutputString to avoid initializing unknown fields string + // unless it is actually needed. For the same reason, disable eager refresh + // on the CodedOutputStream. printer->Print( - " ::google::protobuf::io::StringOutputStream unknown_fields_string(\n" - " mutable_unknown_fields());\n" + " ::google::protobuf::io::LazyStringOutputStream unknown_fields_string(\n" + " ::google::protobuf::internal::NewPermanentCallback(\n" + " &MutableUnknownFieldsFor$classname$, this));\n" " ::google::protobuf::io::CodedOutputStream unknown_fields_stream(\n" - " &unknown_fields_string);\n"); + " &unknown_fields_string, false);\n", + "classname", classname_); } printer->Print( diff --git a/src/google/protobuf/io/coded_stream.cc b/src/google/protobuf/io/coded_stream.cc index 4bcd354f..e3a34d0a 100644 --- a/src/google/protobuf/io/coded_stream.cc +++ b/src/google/protobuf/io/coded_stream.cc @@ -629,6 +629,24 @@ CodedOutputStream::CodedOutputStream(ZeroCopyOutputStream* output) had_error_ = false; } +CodedOutputStream::CodedOutputStream(ZeroCopyOutputStream* output, + bool do_eager_refresh) + : output_(output), + buffer_(NULL), + buffer_size_(0), + total_bytes_(0), + had_error_(false), + aliasing_enabled_(false) { + if (do_eager_refresh) { + // Eagerly Refresh() so buffer space is immediately available. + Refresh(); + // The Refresh() may have failed. If the client doesn't write any data, + // though, don't consider this an error. If the client does write data, then + // another Refresh() will be attempted and it will set the error once again. + had_error_ = false; + } +} + CodedOutputStream::~CodedOutputStream() { Trim(); } diff --git a/src/google/protobuf/io/coded_stream.h b/src/google/protobuf/io/coded_stream.h index 2da096c5..020f20c7 100644 --- a/src/google/protobuf/io/coded_stream.h +++ b/src/google/protobuf/io/coded_stream.h @@ -666,6 +666,7 @@ class LIBPROTOBUF_EXPORT CodedOutputStream { public: // Create an CodedOutputStream that writes to the given ZeroCopyOutputStream. explicit CodedOutputStream(ZeroCopyOutputStream* output); + CodedOutputStream(ZeroCopyOutputStream* output, bool do_eager_refresh); // Destroy the CodedOutputStream and position the underlying // ZeroCopyOutputStream immediately after the last byte written. diff --git a/src/google/protobuf/io/zero_copy_stream_impl_lite.cc b/src/google/protobuf/io/zero_copy_stream_impl_lite.cc index 686e63f2..083beca4 100644 --- a/src/google/protobuf/io/zero_copy_stream_impl_lite.cc +++ b/src/google/protobuf/io/zero_copy_stream_impl_lite.cc @@ -157,6 +157,7 @@ StringOutputStream::~StringOutputStream() { } bool StringOutputStream::Next(void** data, int* size) { + GOOGLE_CHECK_NE(NULL, target_); int old_size = target_->size(); // Grow the string. @@ -188,14 +189,44 @@ bool StringOutputStream::Next(void** data, int* size) { void StringOutputStream::BackUp(int count) { GOOGLE_CHECK_GE(count, 0); + GOOGLE_CHECK_NE(NULL, target_); GOOGLE_CHECK_LE(count, target_->size()); target_->resize(target_->size() - count); } int64 StringOutputStream::ByteCount() const { + GOOGLE_CHECK_NE(NULL, target_); return target_->size(); } +void StringOutputStream::SetString(string* target) { + target_ = target; +} + +// =================================================================== + +LazyStringOutputStream::LazyStringOutputStream( + ResultCallback<string*>* callback) + : StringOutputStream(NULL), + callback_(GOOGLE_CHECK_NOTNULL(callback)), + string_is_set_(false) { +} + +LazyStringOutputStream::~LazyStringOutputStream() { +} + +bool LazyStringOutputStream::Next(void** data, int* size) { + if (!string_is_set_) { + SetString(callback_->Run()); + string_is_set_ = true; + } + return StringOutputStream::Next(data, size); +} + +int64 LazyStringOutputStream::ByteCount() const { + return string_is_set_ ? StringOutputStream::ByteCount() : 0; +} + // =================================================================== CopyingInputStream::~CopyingInputStream() {} diff --git a/src/google/protobuf/io/zero_copy_stream_impl_lite.h b/src/google/protobuf/io/zero_copy_stream_impl_lite.h index a598ef2e..1c397dea 100644 --- a/src/google/protobuf/io/zero_copy_stream_impl_lite.h +++ b/src/google/protobuf/io/zero_copy_stream_impl_lite.h @@ -148,6 +148,9 @@ class LIBPROTOBUF_EXPORT StringOutputStream : public ZeroCopyOutputStream { void BackUp(int count); int64 ByteCount() const; + protected: + void SetString(string* target); + private: static const int kMinimumSize = 16; @@ -156,6 +159,27 @@ class LIBPROTOBUF_EXPORT StringOutputStream : public ZeroCopyOutputStream { GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(StringOutputStream); }; +// LazyStringOutputStream is a StringOutputStream with lazy acquisition of +// the output string from a callback. The string is owned externally, and not +// deleted in the stream destructor. +class LIBPROTOBUF_EXPORT LazyStringOutputStream : public StringOutputStream { + public: + // Callback should be permanent (non-self-deleting). Ownership is transferred + // to the LazyStringOutputStream. + explicit LazyStringOutputStream(ResultCallback<string*>* callback); + ~LazyStringOutputStream(); + + // implements ZeroCopyOutputStream, overriding StringOutputStream ----------- + bool Next(void** data, int* size); + int64 ByteCount() const; + + private: + const scoped_ptr<ResultCallback<string*> > callback_; + bool string_is_set_; + + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(LazyStringOutputStream); +}; + // Note: There is no StringInputStream. Instead, just create an // ArrayInputStream as follows: // ArrayInputStream input(str.data(), str.size()); diff --git a/src/google/protobuf/stubs/callback.h b/src/google/protobuf/stubs/callback.h index 6da530d3..1ba6ea7a 100644 --- a/src/google/protobuf/stubs/callback.h +++ b/src/google/protobuf/stubs/callback.h @@ -78,6 +78,18 @@ class LIBPROTOBUF_EXPORT Closure { GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Closure); }; +template<typename R> +class LIBPROTOBUF_EXPORT ResultCallback { + public: + ResultCallback() {} + virtual ~ResultCallback() {} + + virtual R Run() = 0; + + private: + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ResultCallback); +}; + template<typename R, typename A1> class LIBPROTOBUF_EXPORT ResultCallback1 { public: @@ -240,6 +252,50 @@ class MethodClosure2 : public Closure { Arg2 arg2_; }; +template<typename R> +class FunctionResultCallback_0_0 : public ResultCallback<R> { + public: + typedef R (*FunctionType)(); + + FunctionResultCallback_0_0(FunctionType function, bool self_deleting) + : function_(function), self_deleting_(self_deleting) {} + ~FunctionResultCallback_0_0() {} + + R Run() { + bool needs_delete = self_deleting_; // read in case callback deletes + R result = function_(); + if (needs_delete) delete this; + return result; + } + + private: + FunctionType function_; + bool self_deleting_; +}; + +template<typename R, typename P1> +class FunctionResultCallback_1_0 : public ResultCallback<R> { + public: + typedef R (*FunctionType)(P1); + + FunctionResultCallback_1_0(FunctionType function, bool self_deleting, + P1 p1) + : function_(function), self_deleting_(self_deleting), p1_(p1) {} + ~FunctionResultCallback_1_0() {} + + R Run() { + bool needs_delete = self_deleting_; // read in case callback deletes + R result = function_(p1_); + if (needs_delete) delete this; + return result; + } + + private: + FunctionType function_; + bool self_deleting_; + P1 p1_; +}; + template<typename R, typename Arg1> class FunctionResultCallback_0_1 : public ResultCallback1<R, Arg1> { public: @@ -408,6 +464,33 @@ inline Closure* NewPermanentCallback( object, method, false, arg1, arg2); } +// See ResultCallback +template<typename R> +inline ResultCallback<R>* NewCallback(R (*function)()) { + return new internal::FunctionResultCallback_0_0<R>(function, true); +} + +// See ResultCallback +template<typename R> +inline ResultCallback<R>* NewPermanentCallback(R (*function)()) { + return new internal::FunctionResultCallback_0_0<R>(function, false); +} + +// See ResultCallback +template<typename R, typename P1> +inline ResultCallback<R>* NewCallback(R (*function)(P1), P1 p1) { + return new internal::FunctionResultCallback_1_0<R, P1>( + function, true, p1); +} + +// See ResultCallback +template<typename R, typename P1> +inline ResultCallback<R>* NewPermanentCallback( + R (*function)(P1), P1 p1) { + return new internal::FunctionResultCallback_1_0<R, P1>( + function, false, p1); +} + // See ResultCallback1 template<typename R, typename A1> inline ResultCallback1<R, A1>* NewCallback(R (*function)(A1)) { |