diff options
author | Josh Haberman <jhaberman@gmail.com> | 2016-05-17 18:07:39 -0700 |
---|---|---|
committer | Josh Haberman <jhaberman@gmail.com> | 2016-06-03 09:39:38 -0700 |
commit | bd98eae1c944e453784cdf96c05cc40de55a9690 (patch) | |
tree | 7698c6534763424ffbcb86f84bfeebdadc8d2d0d | |
parent | ef7894e2dc6d287419e42a4fdc52cdfedd386d16 (diff) | |
download | protobuf-bd98eae1c944e453784cdf96c05cc40de55a9690.tar.gz protobuf-bd98eae1c944e453784cdf96c05cc40de55a9690.tar.bz2 protobuf-bd98eae1c944e453784cdf96c05cc40de55a9690.zip |
Fixed Python by updating failure lists and fixed a few broken tests.
Python 2.x doesn't detect unpaired surrogates so we have to
do that manually.
-rw-r--r-- | conformance/Makefile.am | 4 | ||||
-rw-r--r-- | conformance/failure_list_csharp.txt | 5 | ||||
-rw-r--r-- | conformance/failure_list_python.txt | 54 | ||||
-rw-r--r-- | conformance/failure_list_python_cpp.txt | 38 | ||||
-rw-r--r-- | python/google/protobuf/json_format.py | 10 |
5 files changed, 28 insertions, 83 deletions
diff --git a/conformance/Makefile.am b/conformance/Makefile.am index 31a9e408..5538cc75 100644 --- a/conformance/Makefile.am +++ b/conformance/Makefile.am @@ -268,10 +268,10 @@ test_ruby: protoc_middleman conformance-test-runner $(other_language_protoc_outp # These depend on library paths being properly set up. The easiest way to # run them is to just use "tox" from the python dir. test_python: protoc_middleman conformance-test-runner - ./conformance-test-runner --failure_list failure_list_python.txt $(CONFORMANCE_PYTHON_EXTRA_FAILURES) ./conformance_python.py + ./conformance-test-runner --failure_list failure_list_python.txt ./conformance_python.py test_python_cpp: protoc_middleman conformance-test-runner - ./conformance-test-runner --failure_list failure_list_python_cpp.txt $(CONFORMANCE_PYTHON_EXTRA_FAILURES) ./conformance_python.py + ./conformance-test-runner --failure_list failure_list_python_cpp.txt ./conformance_python.py if OBJC_CONFORMANCE_TEST diff --git a/conformance/failure_list_csharp.txt b/conformance/failure_list_csharp.txt index a46cee47..e7de4b96 100644 --- a/conformance/failure_list_csharp.txt +++ b/conformance/failure_list_csharp.txt @@ -1,16 +1,11 @@ -JsonInput.AnyWithValueForInteger.JsonOutput -JsonInput.AnyWithValueForJsonObject.JsonOutput JsonInput.FieldNameInLowerCamelCase.Validator JsonInput.FieldNameInSnakeCase.JsonOutput JsonInput.FieldNameInSnakeCase.ProtobufOutput JsonInput.FieldNameWithMixedCases.JsonOutput JsonInput.FieldNameWithMixedCases.ProtobufOutput JsonInput.FieldNameWithMixedCases.Validator -JsonInput.Int32FieldMinFloatValue.JsonOutput -JsonInput.Int32FieldMinValue.JsonOutput JsonInput.Int64FieldMaxValueNotQuoted.JsonOutput JsonInput.Int64FieldMaxValueNotQuoted.ProtobufOutput JsonInput.OriginalProtoFieldName.JsonOutput -JsonInput.StringFieldSurrogatePair.JsonOutput JsonInput.Uint64FieldMaxValueNotQuoted.JsonOutput JsonInput.Uint64FieldMaxValueNotQuoted.ProtobufOutput diff --git a/conformance/failure_list_python.txt b/conformance/failure_list_python.txt index d2e52637..ab7fc031 100644 --- a/conformance/failure_list_python.txt +++ b/conformance/failure_list_python.txt @@ -3,26 +3,7 @@ DurationProtoInputTooSmall.JsonOutput FieldMaskNumbersDontRoundTrip.JsonOutput FieldMaskPathsDontRoundTrip.JsonOutput FieldMaskTooManyUnderscore.JsonOutput -JsonInput.Any.JsonOutput -JsonInput.Any.ProtobufOutput -JsonInput.AnyNested.JsonOutput -JsonInput.AnyNested.ProtobufOutput -JsonInput.AnyUnorderedTypeTag.JsonOutput -JsonInput.AnyUnorderedTypeTag.ProtobufOutput -JsonInput.AnyWithDuration.JsonOutput -JsonInput.AnyWithDuration.ProtobufOutput -JsonInput.AnyWithFieldMask.JsonOutput JsonInput.AnyWithFieldMask.ProtobufOutput -JsonInput.AnyWithInt32ValueWrapper.JsonOutput -JsonInput.AnyWithInt32ValueWrapper.ProtobufOutput -JsonInput.AnyWithStruct.JsonOutput -JsonInput.AnyWithStruct.ProtobufOutput -JsonInput.AnyWithTimestamp.JsonOutput -JsonInput.AnyWithTimestamp.ProtobufOutput -JsonInput.AnyWithValueForInteger.JsonOutput -JsonInput.AnyWithValueForInteger.ProtobufOutput -JsonInput.AnyWithValueForJsonObject.JsonOutput -JsonInput.AnyWithValueForJsonObject.ProtobufOutput JsonInput.BytesFieldInvalidBase64Characters JsonInput.DoubleFieldInfinityNotQuoted JsonInput.DoubleFieldNanNotQuoted @@ -54,32 +35,29 @@ JsonInput.Int32FieldMaxFloatValue.JsonOutput JsonInput.Int32FieldMaxFloatValue.ProtobufOutput JsonInput.Int32FieldMinFloatValue.JsonOutput JsonInput.Int32FieldMinFloatValue.ProtobufOutput -JsonInput.Int32FieldMinValue.JsonOutput JsonInput.OriginalProtoFieldName.JsonOutput JsonInput.OriginalProtoFieldName.ProtobufOutput -JsonInput.RepeatedFieldMessageElementIsNull -JsonInput.RepeatedFieldPrimitiveElementIsNull JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool -JsonInput.StringFieldSurrogatePair.JsonOutput -JsonInput.StringFieldUnpairedLowSurrogate -JsonInput.Struct.JsonOutput -JsonInput.Struct.ProtobufOutput JsonInput.TimestampJsonInputLowercaseT JsonInput.Uint32FieldMaxFloatValue.JsonOutput JsonInput.Uint32FieldMaxFloatValue.ProtobufOutput -JsonInput.ValueAcceptBool.JsonOutput -JsonInput.ValueAcceptBool.ProtobufOutput -JsonInput.ValueAcceptFloat.JsonOutput -JsonInput.ValueAcceptFloat.ProtobufOutput -JsonInput.ValueAcceptInteger.JsonOutput -JsonInput.ValueAcceptInteger.ProtobufOutput -JsonInput.ValueAcceptList.JsonOutput -JsonInput.ValueAcceptList.ProtobufOutput JsonInput.ValueAcceptNull.JsonOutput JsonInput.ValueAcceptNull.ProtobufOutput -JsonInput.ValueAcceptObject.JsonOutput -JsonInput.ValueAcceptObject.ProtobufOutput -JsonInput.ValueAcceptString.JsonOutput -JsonInput.ValueAcceptString.ProtobufOutput +ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE +ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE +ProtobufInput.PrematureEofInPackedField.BOOL +ProtobufInput.PrematureEofInPackedField.DOUBLE +ProtobufInput.PrematureEofInPackedField.ENUM +ProtobufInput.PrematureEofInPackedField.FIXED32 +ProtobufInput.PrematureEofInPackedField.FIXED64 +ProtobufInput.PrematureEofInPackedField.FLOAT +ProtobufInput.PrematureEofInPackedField.INT32 +ProtobufInput.PrematureEofInPackedField.INT64 +ProtobufInput.PrematureEofInPackedField.SFIXED32 +ProtobufInput.PrematureEofInPackedField.SFIXED64 +ProtobufInput.PrematureEofInPackedField.SINT32 +ProtobufInput.PrematureEofInPackedField.SINT64 +ProtobufInput.PrematureEofInPackedField.UINT32 +ProtobufInput.PrematureEofInPackedField.UINT64 TimestampProtoInputTooLarge.JsonOutput TimestampProtoInputTooSmall.JsonOutput diff --git a/conformance/failure_list_python_cpp.txt b/conformance/failure_list_python_cpp.txt index 7b5e45f9..1eb916ab 100644 --- a/conformance/failure_list_python_cpp.txt +++ b/conformance/failure_list_python_cpp.txt @@ -12,26 +12,7 @@ DurationProtoInputTooSmall.JsonOutput FieldMaskNumbersDontRoundTrip.JsonOutput FieldMaskPathsDontRoundTrip.JsonOutput FieldMaskTooManyUnderscore.JsonOutput -JsonInput.Any.JsonOutput -JsonInput.Any.ProtobufOutput -JsonInput.AnyNested.JsonOutput -JsonInput.AnyNested.ProtobufOutput -JsonInput.AnyUnorderedTypeTag.JsonOutput -JsonInput.AnyUnorderedTypeTag.ProtobufOutput -JsonInput.AnyWithDuration.JsonOutput -JsonInput.AnyWithDuration.ProtobufOutput -JsonInput.AnyWithFieldMask.JsonOutput JsonInput.AnyWithFieldMask.ProtobufOutput -JsonInput.AnyWithInt32ValueWrapper.JsonOutput -JsonInput.AnyWithInt32ValueWrapper.ProtobufOutput -JsonInput.AnyWithStruct.JsonOutput -JsonInput.AnyWithStruct.ProtobufOutput -JsonInput.AnyWithTimestamp.JsonOutput -JsonInput.AnyWithTimestamp.ProtobufOutput -JsonInput.AnyWithValueForInteger.JsonOutput -JsonInput.AnyWithValueForInteger.ProtobufOutput -JsonInput.AnyWithValueForJsonObject.JsonOutput -JsonInput.AnyWithValueForJsonObject.ProtobufOutput JsonInput.BytesFieldInvalidBase64Characters JsonInput.DoubleFieldInfinityNotQuoted JsonInput.DoubleFieldNanNotQuoted @@ -63,33 +44,14 @@ JsonInput.Int32FieldMaxFloatValue.JsonOutput JsonInput.Int32FieldMaxFloatValue.ProtobufOutput JsonInput.Int32FieldMinFloatValue.JsonOutput JsonInput.Int32FieldMinFloatValue.ProtobufOutput -JsonInput.Int32FieldMinValue.JsonOutput JsonInput.OriginalProtoFieldName.JsonOutput JsonInput.OriginalProtoFieldName.ProtobufOutput -JsonInput.RepeatedFieldMessageElementIsNull -JsonInput.RepeatedFieldPrimitiveElementIsNull JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool -JsonInput.StringFieldSurrogatePair.JsonOutput -JsonInput.StringFieldUnpairedLowSurrogate -JsonInput.Struct.JsonOutput -JsonInput.Struct.ProtobufOutput JsonInput.TimestampJsonInputLowercaseT JsonInput.Uint32FieldMaxFloatValue.JsonOutput JsonInput.Uint32FieldMaxFloatValue.ProtobufOutput -JsonInput.ValueAcceptBool.JsonOutput -JsonInput.ValueAcceptBool.ProtobufOutput -JsonInput.ValueAcceptFloat.JsonOutput -JsonInput.ValueAcceptFloat.ProtobufOutput -JsonInput.ValueAcceptInteger.JsonOutput -JsonInput.ValueAcceptInteger.ProtobufOutput -JsonInput.ValueAcceptList.JsonOutput -JsonInput.ValueAcceptList.ProtobufOutput JsonInput.ValueAcceptNull.JsonOutput JsonInput.ValueAcceptNull.ProtobufOutput -JsonInput.ValueAcceptObject.JsonOutput -JsonInput.ValueAcceptObject.ProtobufOutput -JsonInput.ValueAcceptString.JsonOutput -JsonInput.ValueAcceptString.ProtobufOutput ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE ProtobufInput.PrematureEofInPackedField.BOOL diff --git a/python/google/protobuf/json_format.py b/python/google/protobuf/json_format.py index 57aa4077..8af6cd20 100644 --- a/python/google/protobuf/json_format.py +++ b/python/google/protobuf/json_format.py @@ -49,6 +49,7 @@ except ImportError: import base64 import json import math +import re import six import sys @@ -68,6 +69,10 @@ _INFINITY = 'Infinity' _NEG_INFINITY = '-Infinity' _NAN = 'NaN' +if sys.version_info < (3, 0): + _UNPAIRED_SURROGATE_PATTERN = re.compile(six.u( + r'[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]' + )) class Error(Exception): """Top-level module error for json_format.""" @@ -554,6 +559,11 @@ def _ConvertScalarFieldValue(value, field, require_str=False): elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING: if field.type == descriptor.FieldDescriptor.TYPE_BYTES: return base64.b64decode(value) + elif sys.version_info < (3, 0): + # Python 2.x does not detect unpaired surrogates when JSON parsing. + if _UNPAIRED_SURROGATE_PATTERN.search(value): + raise ParseError('Unpaired surrogate') + return value else: return value elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM: |