diff options
Diffstat (limited to 'python/google/protobuf/text_encoding.py')
| -rw-r--r-- | python/google/protobuf/text_encoding.py | 84 |
1 files changed, 47 insertions, 37 deletions
diff --git a/python/google/protobuf/text_encoding.py b/python/google/protobuf/text_encoding.py index 98995638..39898765 100644 --- a/python/google/protobuf/text_encoding.py +++ b/python/google/protobuf/text_encoding.py @@ -33,59 +33,70 @@ import re import six -# Lookup table for utf8 -_cescape_utf8_to_str = [chr(i) for i in range(0, 256)] -_cescape_utf8_to_str[9] = r'\t' # optional escape -_cescape_utf8_to_str[10] = r'\n' # optional escape -_cescape_utf8_to_str[13] = r'\r' # optional escape -_cescape_utf8_to_str[39] = r"\'" # optional escape - -_cescape_utf8_to_str[34] = r'\"' # necessary escape -_cescape_utf8_to_str[92] = r'\\' # necessary escape +_cescape_chr_to_symbol_map = {} +_cescape_chr_to_symbol_map[9] = r'\t' # optional escape +_cescape_chr_to_symbol_map[10] = r'\n' # optional escape +_cescape_chr_to_symbol_map[13] = r'\r' # optional escape +_cescape_chr_to_symbol_map[34] = r'\"' # necessary escape +_cescape_chr_to_symbol_map[39] = r"\'" # optional escape +_cescape_chr_to_symbol_map[92] = r'\\' # necessary escape + +# Lookup table for unicode +_cescape_unicode_to_str = [chr(i) for i in range(0, 256)] +for byte, string in _cescape_chr_to_symbol_map.items(): + _cescape_unicode_to_str[byte] = string # Lookup table for non-utf8, with necessary escapes at (o >= 127 or o < 32) _cescape_byte_to_str = ([r'\%03o' % i for i in range(0, 32)] + [chr(i) for i in range(32, 127)] + [r'\%03o' % i for i in range(127, 256)]) -_cescape_byte_to_str[9] = r'\t' # optional escape -_cescape_byte_to_str[10] = r'\n' # optional escape -_cescape_byte_to_str[13] = r'\r' # optional escape -_cescape_byte_to_str[39] = r"\'" # optional escape - -_cescape_byte_to_str[34] = r'\"' # necessary escape -_cescape_byte_to_str[92] = r'\\' # necessary escape +for byte, string in _cescape_chr_to_symbol_map.items(): + _cescape_byte_to_str[byte] = string +del byte, string def CEscape(text, as_utf8): - """Escape a bytes string for use in an ascii protocol buffer. - - text.encode('string_escape') does not seem to satisfy our needs as it - encodes unprintable characters using two-digit hex escapes whereas our - C++ unescaping function allows hex escapes to be any length. So, - "\0011".encode('string_escape') ends up being "\\x011", which will be - decoded in C++ as a single-character string with char code 0x11. + # type: (...) -> str + """Escape a bytes string for use in an text protocol buffer. Args: - text: A byte string to be escaped - as_utf8: Specifies if result should be returned in UTF-8 encoding + text: A byte string to be escaped. + as_utf8: Specifies if result may contain non-ASCII characters. + In Python 3 this allows unescaped non-ASCII Unicode characters. + In Python 2 the return value will be valid UTF-8 rather than only ASCII. Returns: - Escaped string + Escaped string (str). """ - # PY3 hack: make Ord work for str and bytes: - # //platforms/networking/data uses unicode here, hence basestring. - Ord = ord if isinstance(text, six.string_types) else lambda x: x + # Python's text.encode() 'string_escape' or 'unicode_escape' codecs do not + # satisfy our needs; they encodes unprintable characters using two-digit hex + # escapes whereas our C++ unescaping function allows hex escapes to be any + # length. So, "\0011".encode('string_escape') ends up being "\\x011", which + # will be decoded in C++ as a single-character string with char code 0x11. + if six.PY3: + text_is_unicode = isinstance(text, str) + if as_utf8 and text_is_unicode: + # We're already unicode, no processing beyond control char escapes. + return text.translate(_cescape_chr_to_symbol_map) + ord_ = ord if text_is_unicode else lambda x: x # bytes iterate as ints. + else: + ord_ = ord # PY2 if as_utf8: - return ''.join(_cescape_utf8_to_str[Ord(c)] for c in text) - return ''.join(_cescape_byte_to_str[Ord(c)] for c in text) + return ''.join(_cescape_unicode_to_str[ord_(c)] for c in text) + return ''.join(_cescape_byte_to_str[ord_(c)] for c in text) _CUNESCAPE_HEX = re.compile(r'(\\+)x([0-9a-fA-F])(?![0-9a-fA-F])') -_cescape_highbit_to_str = ([chr(i) for i in range(0, 127)] + - [r'\%03o' % i for i in range(127, 256)]) def CUnescape(text): - """Unescape a text string with C-style escape sequences to UTF-8 bytes.""" + # type: (str) -> bytes + """Unescape a text string with C-style escape sequences to UTF-8 bytes. + + Args: + text: The data to parse in a str. + Returns: + A byte string. + """ def ReplaceHex(m): # Only replace the match if the number of leading back slashes is odd. i.e. @@ -98,10 +109,9 @@ def CUnescape(text): # allow single-digit hex escapes (like '\xf'). result = _CUNESCAPE_HEX.sub(ReplaceHex, text) - if str is bytes: # PY2 + if six.PY2: return result.decode('string_escape') - result = ''.join(_cescape_highbit_to_str[ord(c)] for c in result) - return (result.encode('ascii') # Make it bytes to allow decode. + return (result.encode('utf-8') # PY3: Make it bytes to allow decode. .decode('unicode_escape') # Make it bytes again to return the proper type. .encode('raw_unicode_escape')) |