diff options
71 files changed, 2107 insertions, 1487 deletions
@@ -27,6 +27,7 @@ src/Makefile /config.h config.log config.status +pbconfig.h libtool protobuf-lite.pc diff --git a/.travis.yml b/.travis.yml index 93013b8b..cbddac58 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,19 @@ -language: cpp -script: ./autogen.sh && ./configure && make distcheck +sudo: false +language: java +jdk: + - openjdk6 + - openjdk7 + - oraclejdk7 +os: + - linux + - osx +script: + - ./autogen.sh && ./configure && make -j2 + - cd java && mvn test && cd .. + - cd javanano && mvn test && cd .. + - cd python && python setup.py build && python setup.py test && cd .. + - export LD_LIBRARY_PATH=../src/.libs + - cd python && python setup.py build --cpp_implementation && python setup.py test --cpp_implementation && cd .. + - make distcheck -j2 notifications: email: false diff --git a/Makefile.am b/Makefile.am index 1c039230..4fd93f26 100644 --- a/Makefile.am +++ b/Makefile.am @@ -9,7 +9,7 @@ AUTOMAKE_OPTIONS = foreign SUBDIRS = . src # Always include gtest in distributions. -DIST_SUBDIRS = $(subdirs) src +DIST_SUBDIRS = $(subdirs) src conformance # Build gtest before we build protobuf tests. We don't add gtest to SUBDIRS # because then "make check" would also build and run all of gtest's own tests, @@ -30,6 +30,10 @@ clean-local: @if test -e gtest/Makefile; then \ echo "Making clean in gtest"; \ cd gtest && $(MAKE) $(AM_MAKEFLAGS) clean; \ + fi; \ + if test -e conformance/Makefile; then \ + echo "Making clean in conformance"; \ + cd conformance && $(MAKE) $(AM_MAKEFLAGS) clean; \ fi pkgconfigdir = $(libdir)/pkgconfig @@ -152,7 +156,7 @@ java_EXTRA_DIST= \ java/src/test/java/com/google/protobuf/test_check_utf8_size.proto \ java/src/test/java/com/google/protobuf/test_custom_options.proto \ java/pom.xml \ - java/README.txt + java/README.md javanano_EXTRA_DIST= \ javanano/src/main/java/com/google/protobuf/nano/CodedOutputByteBufferNano.java \ @@ -190,7 +194,7 @@ javanano_EXTRA_DIST= javanano/src/test/java/com/google/protobuf/nano/unittest_multiple_nano.proto \ javanano/src/test/java/com/google/protobuf/nano/unittest_enum_class_multiple_nano.proto \ javanano/src/test/java/com/google/protobuf/nano/map_test.proto \ - javanano/README.txt \ + javanano/README.md \ javanano/pom.xml @@ -198,23 +202,19 @@ python_EXTRA_DIST= \ python/google/protobuf/internal/api_implementation.cc \ python/google/protobuf/internal/api_implementation.py \ python/google/protobuf/internal/containers.py \ - python/google/protobuf/internal/cpp_message.py \ python/google/protobuf/internal/decoder.py \ python/google/protobuf/internal/descriptor_database_test.py \ python/google/protobuf/internal/descriptor_pool_test.py \ python/google/protobuf/internal/descriptor_pool_test1.proto \ python/google/protobuf/internal/descriptor_pool_test2.proto \ - python/google/protobuf/internal/descriptor_python_test.py \ python/google/protobuf/internal/descriptor_test.py \ python/google/protobuf/internal/encoder.py \ python/google/protobuf/internal/enum_type_wrapper.py \ python/google/protobuf/internal/factory_test1.proto \ python/google/protobuf/internal/factory_test2.proto \ python/google/protobuf/internal/generator_test.py \ - python/google/protobuf/internal/message_factory_python_test.py \ python/google/protobuf/internal/message_factory_test.py \ python/google/protobuf/internal/message_listener.py \ - python/google/protobuf/internal/message_python_test.py \ python/google/protobuf/internal/message_test.py \ python/google/protobuf/internal/missing_enum_values.proto \ python/google/protobuf/internal/more_extensions.proto \ @@ -277,7 +277,7 @@ python_EXTRA_DIST= \ python/setup.py \ python/mox.py \ python/stubout.py \ - python/README.txt + python/README.md ruby_EXTRA_DIST= \ ruby/README.md \ diff --git a/configure.ac b/configure.ac index 14351a8b..0615cd85 100644 --- a/configure.ac +++ b/configure.ac @@ -164,5 +164,5 @@ export CFLAGS export CXXFLAGS AC_CONFIG_SUBDIRS([gtest]) -AC_CONFIG_FILES([Makefile src/Makefile protobuf.pc protobuf-lite.pc]) +AC_CONFIG_FILES([Makefile src/Makefile conformance/Makefile protobuf.pc protobuf-lite.pc]) AC_OUTPUT diff --git a/conformance/Makefile.am b/conformance/Makefile.am new file mode 100644 index 00000000..0c4eae75 --- /dev/null +++ b/conformance/Makefile.am @@ -0,0 +1,50 @@ +## Process this file with automake to produce Makefile.in + +protoc_inputs = \ + conformance.proto + +protoc_outputs = \ + conformance.pb.cc \ + conformance.pb.h + +bin_PROGRAMS = conformance-test-runner conformance-cpp + +conformance_test_runner_LDADD = $(top_srcdir)/src/libprotobuf.la +conformance_test_runner_SOURCES = conformance_test.cc conformance_test_runner.cc +nodist_conformance_test_runner_SOURCES = conformance.pb.cc +conformance_test_runner_CPPFLAGS = -I$(top_srcdir)/src + +conformance_cpp_LDADD = $(top_srcdir)/src/libprotobuf.la +conformance_cpp_SOURCES = conformance_cpp.cc +nodist_conformance_cpp_SOURCES = conformance.pb.cc +conformance_cpp_CPPFLAGS = -I$(top_srcdir)/src + +if USE_EXTERNAL_PROTOC + +unittest_proto_middleman: $(protoc_inputs) + $(PROTOC) -I$(srcdir) --cpp_out=. $^ + touch unittest_proto_middleman + +else + +# We have to cd to $(srcdir) before executing protoc because $(protoc_inputs) is +# relative to srcdir, which may not be the same as the current directory when +# building out-of-tree. +unittest_proto_middleman: $(top_srcdir)/src/protoc$(EXEEXT) $(protoc_inputs) + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. --cpp_out=$$oldpwd $(protoc_inputs) ) + touch unittest_proto_middleman + +endif + +$(protoc_outputs): unittest_proto_middleman + +BUILT_SOURCES = $(protoc_outputs) + +CLEANFILES = $(protoc_outputs) unittest_proto_middleman + +MAINTAINERCLEANFILES = \ + Makefile.in + +# Targets for actually running tests. +test_cpp: unittest_proto_middleman conformance-test-runner conformance-cpp + ./conformance-test-runner ./conformance-cpp diff --git a/conformance/README.md b/conformance/README.md new file mode 100644 index 00000000..9388055f --- /dev/null +++ b/conformance/README.md @@ -0,0 +1,45 @@ +Protocol Buffers - Google's data interchange format +=================================================== + +[](https://travis-ci.org/google/protobuf) + +Copyright 2008 Google Inc. + +This directory contains conformance tests for testing completeness and +correctness of Protocol Buffers implementations. These tests are designed +to be easy to run against any Protocol Buffers implementation. + +This directory contains the tester process `conformance-test`, which +contains all of the tests themselves. Then separate programs written +in whatever language you want to test communicate with the tester +program over a pipe. + +Before running any of these tests, make sure you run `make` in the base +directory to build `protoc`, since all the tests depend on it. + + $ make + +Then to run the tests against the C++ implementation, run: + + $ cd conformance && make test_cpp + +More tests and languages will be added soon! + +Testing other Protocol Buffer implementations +--------------------------------------------- + +To run these tests against a new Protocol Buffers implementation, write a +program in your language that uses the protobuf implementation you want +to test. This program should implement the testing protocol defined in +[conformance.proto](https://github.com/google/protobuf/blob/master/conformance/conformance.proto). +This is designed to be as easy as possible: the C++ version is only +150 lines and is a good example for what this program should look like +(see [conformance_cpp.cc](https://github.com/google/protobuf/blob/master/conformance/conformance_cpp.cc)). +The program only needs to be able to read from stdin and write to stdout. + +Portability +----------- + +Note that the test runner currently does not work on Windows. Patches +to fix this are welcome! (But please get in touch first to settle on +a general implementation strategy). diff --git a/conformance/conformance.proto b/conformance/conformance.proto new file mode 100644 index 00000000..0fb66cf9 --- /dev/null +++ b/conformance/conformance.proto @@ -0,0 +1,215 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; +package conformance; + +// This defines the conformance testing protocol. This protocol exists between +// the conformance test suite itself and the code being tested. For each test, +// the suite will send a ConformanceRequest message and expect a +// ConformanceResponse message. +// +// You can either run the tests in two different ways: +// +// 1. in-process (using the interface in conformance_test.h). +// +// 2. as a sub-process communicating over a pipe. Information about how to +// do this is in conformance_test_runner.cc. +// +// Pros/cons of the two approaches: +// +// - running as a sub-process is much simpler for languages other than C/C++. +// +// - running as a sub-process may be more tricky in unusual environments like +// iOS apps, where fork/stdin/stdout are not available. + +// Represents a single test case's input. The testee should: +// +// 1. parse this proto (which should always succeed) +// 2. parse the protobuf or JSON payload in "payload" (which may fail) +// 3. if the parse succeeded, serialize the message in the requested format. +message ConformanceRequest { + // The payload (whether protobuf of JSON) is always for a TestAllTypes proto + // (see below). + oneof payload { + bytes protobuf_payload = 1; + string json_payload = 2; + } + + enum RequestedOutput { + UNSPECIFIED = 0; + PROTOBUF = 1; + JSON = 2; + } + + // Which format should the testee serialize its message to? + optional RequestedOutput requested_output = 3; +} + +// Represents a single test case's output. +message ConformanceResponse { + oneof result { + // This string should be set to indicate parsing failed. The string can + // provide more information about the parse error if it is available. + // + // Setting this string does not necessarily mean the testee failed the + // test. Some of the test cases are intentionally invalid input. + string parse_error = 1; + + // This should be set if some other error occurred. This will always + // indicate that the test failed. The string can provide more information + // about the failure. + string runtime_error = 2; + + // If the input was successfully parsed and the requested output was + // protobuf, serialize it to protobuf and set it in this field. + bytes protobuf_payload = 3; + + // If the input was successfully parsed and the requested output was JSON, + // serialize to JSON and set it in this field. + string json_payload = 4; + } +} + +// This proto includes every type of field in both singular and repeated +// forms. +message TestAllTypes { + message NestedMessage { + optional int32 a = 1; + optional TestAllTypes corecursive = 2; + } + + enum NestedEnum { + FOO = 0; + BAR = 1; + BAZ = 2; + NEG = -1; // Intentionally negative. + } + + // Singular + optional int32 optional_int32 = 1; + optional int64 optional_int64 = 2; + optional uint32 optional_uint32 = 3; + optional uint64 optional_uint64 = 4; + optional sint32 optional_sint32 = 5; + optional sint64 optional_sint64 = 6; + optional fixed32 optional_fixed32 = 7; + optional fixed64 optional_fixed64 = 8; + optional sfixed32 optional_sfixed32 = 9; + optional sfixed64 optional_sfixed64 = 10; + optional float optional_float = 11; + optional double optional_double = 12; + optional bool optional_bool = 13; + optional string optional_string = 14; + optional bytes optional_bytes = 15; + + optional group OptionalGroup = 16 { + optional int32 a = 17; + } + + optional NestedMessage optional_nested_message = 18; + optional ForeignMessage optional_foreign_message = 19; + + optional NestedEnum optional_nested_enum = 21; + optional ForeignEnum optional_foreign_enum = 22; + + optional string optional_string_piece = 24 [ctype=STRING_PIECE]; + optional string optional_cord = 25 [ctype=CORD]; + + optional TestAllTypes recursive_message = 27; + + // Repeated + repeated int32 repeated_int32 = 31; + repeated int64 repeated_int64 = 32; + repeated uint32 repeated_uint32 = 33; + repeated uint64 repeated_uint64 = 34; + repeated sint32 repeated_sint32 = 35; + repeated sint64 repeated_sint64 = 36; + repeated fixed32 repeated_fixed32 = 37; + repeated fixed64 repeated_fixed64 = 38; + repeated sfixed32 repeated_sfixed32 = 39; + repeated sfixed64 repeated_sfixed64 = 40; + repeated float repeated_float = 41; + repeated double repeated_double = 42; + repeated bool repeated_bool = 43; + repeated string repeated_string = 44; + repeated bytes repeated_bytes = 45; + + repeated group RepeatedGroup = 46 { + optional int32 a = 47; + } + + repeated NestedMessage repeated_nested_message = 48; + repeated ForeignMessage repeated_foreign_message = 49; + + repeated NestedEnum repeated_nested_enum = 51; + repeated ForeignEnum repeated_foreign_enum = 52; + + repeated string repeated_string_piece = 54 [ctype=STRING_PIECE]; + repeated string repeated_cord = 55 [ctype=CORD]; + + // Map + map < int32, int32> map_int32_int32 = 56; + map < int64, int64> map_int64_int64 = 57; + map < uint32, uint32> map_uint32_uint32 = 58; + map < uint64, uint64> map_uint64_uint64 = 59; + map < sint32, sint32> map_sint32_sint32 = 60; + map < sint64, sint64> map_sint64_sint64 = 61; + map < fixed32, fixed32> map_fixed32_fixed32 = 62; + map < fixed64, fixed64> map_fixed64_fixed64 = 63; + map <sfixed32, sfixed32> map_sfixed32_sfixed32 = 64; + map <sfixed64, sfixed64> map_sfixed64_sfixed64 = 65; + map < int32, float> map_int32_float = 66; + map < int32, double> map_int32_double = 67; + map < bool, bool> map_bool_bool = 68; + map < string, string> map_string_string = 69; + map < string, bytes> map_string_bytes = 70; + map < string, NestedMessage> map_string_nested_message = 71; + map < string, ForeignMessage> map_string_foreign_message = 72; + map < string, NestedEnum> map_string_nested_enum = 73; + map < string, ForeignEnum> map_string_foreign_enum = 74; + + oneof oneof_field { + uint32 oneof_uint32 = 111; + NestedMessage oneof_nested_message = 112; + string oneof_string = 113; + bytes oneof_bytes = 114; + } +} + +message ForeignMessage { + optional int32 c = 1; +} + +enum ForeignEnum { + FOREIGN_FOO = 0; + FOREIGN_BAR = 1; + FOREIGN_BAZ = 2; +} diff --git a/conformance/conformance_cpp.cc b/conformance/conformance_cpp.cc new file mode 100644 index 00000000..ff47fbbf --- /dev/null +++ b/conformance/conformance_cpp.cc @@ -0,0 +1,156 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <errno.h> +#include <stdarg.h> +#include <unistd.h> + +#include "conformance.pb.h" + +using std::string; +using conformance::ConformanceRequest; +using conformance::ConformanceResponse; +using conformance::TestAllTypes; + +int test_count = 0; +bool verbose = false; + +bool CheckedRead(int fd, void *buf, size_t len) { + size_t ofs = 0; + while (len > 0) { + ssize_t bytes_read = read(fd, (char*)buf + ofs, len); + + if (bytes_read == 0) return false; + + if (bytes_read < 0) { + GOOGLE_LOG(FATAL) << "Error reading from test runner: " << strerror(errno); + } + + len -= bytes_read; + ofs += bytes_read; + } + + return true; +} + +void CheckedWrite(int fd, const void *buf, size_t len) { + if (write(fd, buf, len) != len) { + GOOGLE_LOG(FATAL) << "Error writing to test runner: " << strerror(errno); + } +} + +void DoTest(const ConformanceRequest& request, ConformanceResponse* response) { + TestAllTypes test_message; + + switch (request.payload_case()) { + case ConformanceRequest::kProtobufPayload: + if (!test_message.ParseFromString(request.protobuf_payload())) { + // Getting parse details would involve something like: + // http://stackoverflow.com/questions/22121922/how-can-i-get-more-details-about-errors-generated-during-protobuf-parsing-c + response->set_parse_error("Parse error (no more details available)."); + return; + } + break; + + case ConformanceRequest::kJsonPayload: + response->set_runtime_error("JSON input is not yet supported."); + break; + + case ConformanceRequest::PAYLOAD_NOT_SET: + GOOGLE_LOG(FATAL) << "Request didn't have payload."; + break; + } + + switch (request.requested_output()) { + case ConformanceRequest::UNSPECIFIED: + GOOGLE_LOG(FATAL) << "Unspecified output format"; + break; + + case ConformanceRequest::PROTOBUF: + test_message.SerializeToString(response->mutable_protobuf_payload()); + break; + + case ConformanceRequest::JSON: + response->set_runtime_error("JSON output is not yet supported."); + break; + } +} + +bool DoTestIo() { + string serialized_input; + string serialized_output; + ConformanceRequest request; + ConformanceResponse response; + uint32_t bytes; + + if (!CheckedRead(STDIN_FILENO, &bytes, sizeof(uint32_t))) { + // EOF. + return false; + } + + serialized_input.resize(bytes); + + if (!CheckedRead(STDIN_FILENO, (char*)serialized_input.c_str(), bytes)) { + GOOGLE_LOG(ERROR) << "Unexpected EOF on stdin. " << strerror(errno); + } + + if (!request.ParseFromString(serialized_input)) { + GOOGLE_LOG(FATAL) << "Parse of ConformanceRequest proto failed."; + return false; + } + + DoTest(request, &response); + + response.SerializeToString(&serialized_output); + + bytes = serialized_output.size(); + CheckedWrite(STDOUT_FILENO, &bytes, sizeof(uint32_t)); + CheckedWrite(STDOUT_FILENO, serialized_output.c_str(), bytes); + + if (verbose) { + fprintf(stderr, "conformance-cpp: request=%s, response=%s\n", + request.ShortDebugString().c_str(), + response.ShortDebugString().c_str()); + } + + test_count++; + + return true; +} + +int main() { + while (1) { + if (!DoTestIo()) { + fprintf(stderr, "conformance-cpp: received EOF from test runner " + "after %d tests, exiting\n", test_count); + return 0; + } + } +} diff --git a/conformance/conformance_test.cc b/conformance/conformance_test.cc new file mode 100644 index 00000000..61029e44 --- /dev/null +++ b/conformance/conformance_test.cc @@ -0,0 +1,310 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <stdarg.h> +#include <string> + +#include "conformance.pb.h" +#include "conformance_test.h" +#include <google/protobuf/stubs/common.h> +#include <google/protobuf/stubs/stringprintf.h> +#include <google/protobuf/wire_format_lite.h> + +using conformance::ConformanceRequest; +using conformance::ConformanceResponse; +using conformance::TestAllTypes; +using google::protobuf::Descriptor; +using google::protobuf::FieldDescriptor; +using google::protobuf::internal::WireFormatLite; +using std::string; + +namespace { + +/* Routines for building arbitrary protos *************************************/ + +// We would use CodedOutputStream except that we want more freedom to build +// arbitrary protos (even invalid ones). + +const string empty; + +string cat(const string& a, const string& b, + const string& c = empty, + const string& d = empty, + const string& e = empty, + const string& f = empty, + const string& g = empty, + const string& h = empty, + const string& i = empty, + const string& j = empty, + const string& k = empty, + const string& l = empty) { + string ret; + ret.reserve(a.size() + b.size() + c.size() + d.size() + e.size() + f.size() + + g.size() + h.size() + i.size() + j.size() + k.size() + l.size()); + ret.append(a); + ret.append(b); + ret.append(c); + ret.append(d); + ret.append(e); + ret.append(f); + ret.append(g); + ret.append(h); + ret.append(i); + ret.append(j); + ret.append(k); + ret.append(l); + return ret; +} + +// The maximum number of bytes that it takes to encode a 64-bit varint. +#define VARINT_MAX_LEN 10 + +size_t vencode64(uint64_t val, char *buf) { + if (val == 0) { buf[0] = 0; return 1; } + size_t i = 0; + while (val) { + uint8_t byte = val & 0x7fU; + val >>= 7; + if (val) byte |= 0x80U; + buf[i++] = byte; + } + return i; +} + +string varint(uint64_t x) { + char buf[VARINT_MAX_LEN]; + size_t len = vencode64(x, buf); + return string(buf, len); +} + +// TODO: proper byte-swapping for big-endian machines. +string fixed32(void *data) { return string(static_cast<char*>(data), 4); } +string fixed64(void *data) { return string(static_cast<char*>(data), 8); } + +string delim(const string& buf) { return cat(varint(buf.size()), buf); } +string uint32(uint32_t u32) { return fixed32(&u32); } +string uint64(uint64_t u64) { return fixed64(&u64); } +string flt(float f) { return fixed32(&f); } +string dbl(double d) { return fixed64(&d); } +string zz32(int32_t x) { return varint(WireFormatLite::ZigZagEncode32(x)); } +string zz64(int64_t x) { return varint(WireFormatLite::ZigZagEncode64(x)); } + +string tag(uint32_t fieldnum, char wire_type) { + return varint((fieldnum << 3) | wire_type); +} + +string submsg(uint32_t fn, const string& buf) { + return cat( tag(fn, WireFormatLite::WIRETYPE_LENGTH_DELIMITED), delim(buf) ); +} + +#define UNKNOWN_FIELD 666 + +uint32_t GetFieldNumberForType(WireFormatLite::FieldType type, bool repeated) { + const Descriptor* d = TestAllTypes().GetDescriptor(); + for (int i = 0; i < d->field_count(); i++) { + const FieldDescriptor* f = d->field(i); + if (static_cast<WireFormatLite::FieldType>(f->type()) == type && + f->is_repeated() == repeated) { + return f->number(); + } + } + GOOGLE_LOG(FATAL) << "Couldn't find field with type " << (int)type; + return 0; +} + +} // anonymous namespace + +namespace google { +namespace protobuf { + +void ConformanceTestSuite::ReportSuccess() { + successes_++; +} + +void ConformanceTestSuite::ReportFailure(const char *fmt, ...) { + va_list args; + va_start(args, fmt); + StringAppendV(&output_, fmt, args); + va_end(args); + failures_++; +} + +void ConformanceTestSuite::RunTest(const ConformanceRequest& request, + ConformanceResponse* response) { + string serialized_request; + string serialized_response; + request.SerializeToString(&serialized_request); + + runner_->RunTest(serialized_request, &serialized_response); + + if (!response->ParseFromString(serialized_response)) { + response->Clear(); + response->set_runtime_error("response proto could not be parsed."); + } + + if (verbose_) { + StringAppendF(&output_, "conformance test: request=%s, response=%s\n", + request.ShortDebugString().c_str(), + response->ShortDebugString().c_str()); + } +} + +void ConformanceTestSuite::DoExpectParseFailureForProto(const string& proto, + int line) { + ConformanceRequest request; + ConformanceResponse response; + request.set_protobuf_payload(proto); + + // We don't expect output, but if the program erroneously accepts the protobuf + // we let it send its response as this. We must not leave it unspecified. + request.set_requested_output(ConformanceRequest::PROTOBUF); + + RunTest(request, &response); + if (response.result_case() == ConformanceResponse::kParseError) { + ReportSuccess(); + } else { + ReportFailure("Should have failed, but didn't. Line: %d, Request: %s, " + "response: %s\n", + line, + request.ShortDebugString().c_str(), + response.ShortDebugString().c_str()); + } +} + +// Expect that this precise protobuf will cause a parse error. +#define ExpectParseFailureForProto(proto) DoExpectParseFailureForProto(proto, __LINE__) + +// Expect that this protobuf will cause a parse error, even if it is followed +// by valid protobuf data. We can try running this twice: once with this +// data verbatim and once with this data followed by some valid data. +// +// TODO(haberman): implement the second of these. +#define ExpectHardParseFailureForProto(proto) DoExpectParseFailureForProto(proto, __LINE__) + + +void ConformanceTestSuite::TestPrematureEOFForType( + WireFormatLite::FieldType type) { + // Incomplete values for each wire type. + static const string incompletes[6] = { + string("\x80"), // VARINT + string("abcdefg"), // 64BIT + string("\x80"), // DELIMITED (partial length) + string(), // START_GROUP (no value required) + string(), // END_GROUP (no value required) + string("abc") // 32BIT + }; + + uint32_t fieldnum = GetFieldNumberForType(type, false); + uint32_t rep_fieldnum = GetFieldNumberForType(type, true); + WireFormatLite::WireType wire_type = + WireFormatLite::WireTypeForFieldType(type); + const string& incomplete = incompletes[wire_type]; + + // EOF before a known non-repeated value. + ExpectParseFailureForProto(tag(fieldnum, wire_type)); + + // EOF before a known repeated value. + ExpectParseFailureForProto(tag(rep_fieldnum, wire_type)); + + // EOF before an unknown value. + ExpectParseFailureForProto(tag(UNKNOWN_FIELD, wire_type)); + + // EOF inside a known non-repeated value. + ExpectParseFailureForProto( + cat( tag(fieldnum, wire_type), incomplete )); + + // EOF inside a known repeated value. + ExpectParseFailureForProto( + cat( tag(rep_fieldnum, wire_type), incomplete )); + + // EOF inside an unknown value. + ExpectParseFailureForProto( + cat( tag(UNKNOWN_FIELD, wire_type), incomplete )); + + if (wire_type == WireFormatLite::WIRETYPE_LENGTH_DELIMITED) { + // EOF in the middle of delimited data for known non-repeated value. + ExpectParseFailureForProto( + cat( tag(fieldnum, wire_type), varint(1) )); + + // EOF in the middle of delimited data for known repeated value. + ExpectParseFailureForProto( + cat( tag(rep_fieldnum, wire_type), varint(1) )); + + // EOF in the middle of delimited data for unknown value. + ExpectParseFailureForProto( + cat( tag(UNKNOWN_FIELD, wire_type), varint(1) )); + + if (type == WireFormatLite::TYPE_MESSAGE) { + // Submessage ends in the middle of a value. + string incomplete_submsg = + cat( tag(WireFormatLite::TYPE_INT32, WireFormatLite::WIRETYPE_VARINT), + incompletes[WireFormatLite::WIRETYPE_VARINT] ); + ExpectHardParseFailureForProto( + cat( tag(fieldnum, WireFormatLite::WIRETYPE_LENGTH_DELIMITED), + varint(incomplete_submsg.size()), + incomplete_submsg )); + } + } else if (type != WireFormatLite::TYPE_GROUP) { + // Non-delimited, non-group: eligible for packing. + + // Packed region ends in the middle of a value. + ExpectHardParseFailureForProto( + cat( tag(rep_fieldnum, WireFormatLite::WIRETYPE_LENGTH_DELIMITED), + varint(incomplete.size()), + incomplete )); + + // EOF in the middle of packed region. + ExpectParseFailureForProto( + cat( tag(rep_fieldnum, WireFormatLite::WIRETYPE_LENGTH_DELIMITED), + varint(1) )); + } +} + +void ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner, + std::string* output) { + runner_ = runner; + output_.clear(); + successes_ = 0; + failures_ = 0; + + for (int i = 1; i <= FieldDescriptor::MAX_TYPE; i++) { + TestPrematureEOFForType(static_cast<WireFormatLite::FieldType>(i)); + } + + StringAppendF(&output_, + "CONFORMANCE SUITE FINISHED: completed %d tests, %d successes, " + "%d failures.\n", + successes_ + failures_, successes_, failures_); + + output->assign(output_); +} + +} // namespace protobuf +} // namespace google diff --git a/conformance/conformance_test.h b/conformance/conformance_test.h new file mode 100644 index 00000000..c16f9c0b --- /dev/null +++ b/conformance/conformance_test.h @@ -0,0 +1,109 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +// This file defines a protocol for running the conformance test suite +// in-process. In other words, the suite itself will run in the same process as +// the code under test. +// +// For pros and cons of this approach, please see conformance.proto. + +#ifndef CONFORMANCE_CONFORMANCE_TEST_H +#define CONFORMANCE_CONFORMANCE_TEST_H + +#include <string> +#include <google/protobuf/wire_format_lite.h> + +namespace conformance { +class ConformanceRequest; +class ConformanceResponse; +} // namespace conformance + +namespace google { +namespace protobuf { + +class ConformanceTestRunner { + public: + // Call to run a single conformance test. + // + // "input" is a serialized conformance.ConformanceRequest. + // "output" should be set to a serialized conformance.ConformanceResponse. + // + // If there is any error in running the test itself, set "runtime_error" in + // the response. + virtual void RunTest(const std::string& input, std::string* output) = 0; +}; + +// Class representing the test suite itself. To run it, implement your own +// class derived from ConformanceTestRunner and then write code like: +// +// class MyConformanceTestRunner : public ConformanceTestRunner { +// public: +// virtual void RunTest(...) { +// // INSERT YOUR FRAMEWORK-SPECIFIC CODE HERE. +// } +// }; +// +// int main() { +// MyConformanceTestRunner runner; +// google::protobuf::ConformanceTestSuite suite; +// +// std::string output; +// suite.RunSuite(&runner, &output); +// } +// +class ConformanceTestSuite { + public: + ConformanceTestSuite() : verbose_(false) {} + + // Run all the conformance tests against the given test runner. + // Test output will be stored in "output". + void RunSuite(ConformanceTestRunner* runner, std::string* output); + + private: + void ReportSuccess(); + void ReportFailure(const char* fmt, ...); + void RunTest(const conformance::ConformanceRequest& request, + conformance::ConformanceResponse* response); + void DoExpectParseFailureForProto(const std::string& proto, int line); + void TestPrematureEOFForType( + google::protobuf::internal::WireFormatLite::FieldType type); + + ConformanceTestRunner* runner_; + int successes_; + int failures_; + bool verbose_; + std::string output_; +}; + +} // namespace protobuf +} // namespace google + +#endif // CONFORMANCE_CONFORMANCE_TEST_H diff --git a/conformance/conformance_test_runner.cc b/conformance/conformance_test_runner.cc new file mode 100644 index 00000000..21277700 --- /dev/null +++ b/conformance/conformance_test_runner.cc @@ -0,0 +1,196 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// This file contains a program for running the test suite in a separate +// process. The other alternative is to run the suite in-process. See +// conformance.proto for pros/cons of these two options. +// +// This program will fork the process under test and communicate with it over +// its stdin/stdout: +// +// +--------+ pipe +----------+ +// | tester | <------> | testee | +// | | | | +// | C++ | | any lang | +// +--------+ +----------+ +// +// The tester contains all of the test cases and their expected output. +// The testee is a simple program written in the target language that reads +// each test case and attempts to produce acceptable output for it. +// +// Every test consists of a ConformanceRequest/ConformanceResponse +// request/reply pair. The protocol on the pipe is simply: +// +// 1. tester sends 4-byte length N +// 2. tester sends N bytes representing a ConformanceRequest proto +// 3. testee sends 4-byte length M +// 4. testee sends M bytes representing a ConformanceResponse proto + +#include <errno.h> +#include <unistd.h> + +#include "conformance.pb.h" +#include "conformance_test.h" + +using conformance::ConformanceRequest; +using conformance::ConformanceResponse; +using google::protobuf::internal::scoped_array; + +#define STRINGIFY(x) #x +#define TOSTRING(x) STRINGIFY(x) +#define CHECK_SYSCALL(call) \ + if (call < 0) { \ + perror(#call " " __FILE__ ":" TOSTRING(__LINE__)); \ + exit(1); \ + } + +// Test runner that spawns the process being tested and communicates with it +// over a pipe. +class ForkPipeRunner : public google::protobuf::ConformanceTestRunner { + public: + ForkPipeRunner(const std::string &executable) + : executable_(executable), running_(false) {} + + void RunTest(const std::string& request, std::string* response) { + if (!running_) { + SpawnTestProgram(); + } + + uint32_t len = request.size(); + CheckedWrite(write_fd_, &len, sizeof(uint32_t)); + CheckedWrite(write_fd_, request.c_str(), request.size()); + CheckedRead(read_fd_, &len, sizeof(uint32_t)); + response->resize(len); + CheckedRead(read_fd_, (void*)response->c_str(), len); + } + + private: + // TODO(haberman): make this work on Windows, instead of using these + // UNIX-specific APIs. + // + // There is a platform-agnostic API in + // src/google/protobuf/compiler/subprocess.h + // + // However that API only supports sending a single message to the subprocess. + // We really want to be able to send messages and receive responses one at a + // time: + // + // 1. Spawning a new process for each test would take way too long for thousands + // of tests and subprocesses like java that can take 100ms or more to start + // up. + // + // 2. Sending all the tests in one big message and receiving all results in one + // big message would take away our visibility about which test(s) caused a + // crash or other fatal error. It would also give us only a single failure + // instead of all of them. + void SpawnTestProgram() { + int toproc_pipe_fd[2]; + int fromproc_pipe_fd[2]; + if (pipe(toproc_pipe_fd) < 0 || pipe(fromproc_pipe_fd) < 0) { + perror("pipe"); + exit(1); + } + + pid_t pid = fork(); + if (pid < 0) { + perror("fork"); + exit(1); + } + + if (pid) { + // Parent. + CHECK_SYSCALL(close(toproc_pipe_fd[0])); + CHECK_SYSCALL(close(fromproc_pipe_fd[1])); + write_fd_ = toproc_pipe_fd[1]; + read_fd_ = fromproc_pipe_fd[0]; + running_ = true; + } else { + // Child. + CHECK_SYSCALL(close(STDIN_FILENO)); + CHECK_SYSCALL(close(STDOUT_FILENO)); + CHECK_SYSCALL(dup2(toproc_pipe_fd[0], STDIN_FILENO)); + CHECK_SYSCALL(dup2(fromproc_pipe_fd[1], STDOUT_FILENO)); + + CHECK_SYSCALL(close(toproc_pipe_fd[0])); + CHECK_SYSCALL(close(fromproc_pipe_fd[1])); + CHECK_SYSCALL(close(toproc_pipe_fd[1])); + CHECK_SYSCALL(close(fromproc_pipe_fd[0])); + + scoped_array<char> executable(new char[executable_.size()]); + memcpy(executable.get(), executable_.c_str(), executable_.size()); + + char *const argv[] = {executable.get(), NULL}; + CHECK_SYSCALL(execv(executable.get(), argv)); // Never returns. + } + } + + void CheckedWrite(int fd, const void *buf, size_t len) { + if (write(fd, buf, len) != len) { + GOOGLE_LOG(FATAL) << "Error writing to test program: " << strerror(errno); + } + } + + void CheckedRead(int fd, void *buf, size_t len) { + size_t ofs = 0; + while (len > 0) { + ssize_t bytes_read = read(fd, (char*)buf + ofs, len); + + if (bytes_read == 0) { + GOOGLE_LOG(FATAL) << "Unexpected EOF from test program"; + } else if (bytes_read < 0) { + GOOGLE_LOG(FATAL) << "Error reading from test program: " << strerror(errno); + } + + len -= bytes_read; + ofs += bytes_read; + } + } + + int write_fd_; + int read_fd_; + bool running_; + std::string executable_; +}; + + +int main(int argc, char *argv[]) { + if (argc < 2) { + fprintf(stderr, "Usage: conformance-test-runner <test-program>\n"); + exit(1); + } + + ForkPipeRunner runner(argv[1]); + google::protobuf::ConformanceTestSuite suite; + + std::string output; + suite.RunSuite(&runner, &output); + + fwrite(output.c_str(), 1, output.size(), stderr); +} diff --git a/java/README.txt b/java/README.md index 57e26a96..060d9ac6 100644 --- a/java/README.txt +++ b/java/README.md @@ -1,4 +1,8 @@ Protocol Buffers - Google's data interchange format +=================================================== + +[](https://travis-ci.org/google/protobuf) + Copyright 2008 Google Inc. This directory contains the Java Protocol Buffers runtime library. @@ -50,17 +54,19 @@ http://maven.apache.org/guides/introduction/introduction-to-profiles.html) E.g. to install the lite version of the jar, you would run: - $ mvn install -P lite + $ mvn install -P lite The resulting artifact has the 'lite' classifier. To reference it for dependency resolution, you would specify it as: +``` <dependency> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> <version>${version}</version> <classifier>lite</classifier> </dependency> +``` Installation - Without Maven ============================ diff --git a/java/src/main/java/com/google/protobuf/ByteString.java b/java/src/main/java/com/google/protobuf/ByteString.java index 9b0a524b..1d5d4e8a 100644 --- a/java/src/main/java/com/google/protobuf/ByteString.java +++ b/java/src/main/java/com/google/protobuf/ByteString.java @@ -37,6 +37,8 @@ import java.io.OutputStream; import java.io.Serializable; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.UnsupportedCharsetException; import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; @@ -76,9 +78,6 @@ public abstract class ByteString implements Iterable<Byte>, Serializable { static final int MIN_READ_FROM_CHUNK_SIZE = 0x100; // 256b static final int MAX_READ_FROM_CHUNK_SIZE = 0x2000; // 8k - // Defined by java.nio.charset.Charset - protected static final String UTF_8 = "UTF-8"; - /** * Empty {@code ByteString}. */ @@ -262,6 +261,18 @@ public abstract class ByteString implements Iterable<Byte>, Serializable { } /** + * Encodes {@code text} into a sequence of bytes using the named charset + * and returns the result as a {@code ByteString}. + * + * @param text source string + * @param charset encode using this charset + * @return new {@code ByteString} + */ + public static ByteString copyFrom(String text, Charset charset) { + return new LiteralByteString(text.getBytes(charset)); + } + + /** * Encodes {@code text} into a sequence of UTF-8 bytes and returns the * result as a {@code ByteString}. * @@ -269,11 +280,7 @@ public abstract class ByteString implements Iterable<Byte>, Serializable { * @return new {@code ByteString} */ public static ByteString copyFromUtf8(String text) { - try { - return new LiteralByteString(text.getBytes(UTF_8)); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException("UTF-8 not supported?", e); - } + return new LiteralByteString(text.getBytes(Internal.UTF_8)); } // ================================================================= @@ -507,7 +514,7 @@ public abstract class ByteString implements Iterable<Byte>, Serializable { /** * Internal (package private) implementation of * {@link #copyTo(byte[],int,int,int)}. - * It assumes that all error checking has already been performed and that + * It assumes that all error checking has already been performed and that * {@code numberToCopy > 0}. */ protected abstract void copyToInternal(byte[] target, int sourceOffset, @@ -546,7 +553,7 @@ public abstract class ByteString implements Iterable<Byte>, Serializable { * @throws IOException if an I/O error occurs. */ public abstract void writeTo(OutputStream out) throws IOException; - + /** * Writes a specified part of this byte string to an output stream. * @@ -572,7 +579,7 @@ public abstract class ByteString implements Iterable<Byte>, Serializable { if (numberToWrite > 0) { writeToInternal(out, sourceOffset, numberToWrite); } - + } /** @@ -599,7 +606,7 @@ public abstract class ByteString implements Iterable<Byte>, Serializable { * <p> * By returning a list, implementations of this method may be able to avoid * copying even when there are multiple backing arrays. - * + * * @return a list of wrapped bytes */ public abstract List<ByteBuffer> asReadOnlyByteBufferList(); @@ -612,8 +619,36 @@ public abstract class ByteString implements Iterable<Byte>, Serializable { * @return new string * @throws UnsupportedEncodingException if charset isn't recognized */ - public abstract String toString(String charsetName) - throws UnsupportedEncodingException; + public String toString(String charsetName) + throws UnsupportedEncodingException { + try { + return toString(Charset.forName(charsetName)); + } catch (UnsupportedCharsetException e) { + UnsupportedEncodingException exception = new UnsupportedEncodingException(charsetName); + exception.initCause(e); + throw exception; + } + } + + /** + * Constructs a new {@code String} by decoding the bytes using the + * specified charset. Returns the same empty String if empty. + * + * @param charset encode using this charset + * @return new string + */ + public String toString(Charset charset) { + return size() == 0 ? "" : toStringInternal(charset); + } + + /** + * Constructs a new {@code String} by decoding the bytes using the + * specified charset. + * + * @param charset encode using this charset + * @return new string + */ + protected abstract String toStringInternal(Charset charset); // ================================================================= // UTF-8 decoding @@ -624,11 +659,7 @@ public abstract class ByteString implements Iterable<Byte>, Serializable { * @return new string using UTF-8 encoding */ public String toStringUtf8() { - try { - return toString(UTF_8); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException("UTF-8 not supported?", e); - } + return toString(Internal.UTF_8); } /** @@ -831,7 +862,7 @@ public abstract class ByteString implements Iterable<Byte>, Serializable { flushLastBuffer(); return ByteString.copyFrom(flushedBuffers); } - + /** * Implement java.util.Arrays.copyOf() for jdk 1.5. */ diff --git a/java/src/main/java/com/google/protobuf/CodedInputStream.java b/java/src/main/java/com/google/protobuf/CodedInputStream.java index 0ca00bab..d201f7c5 100644 --- a/java/src/main/java/com/google/protobuf/CodedInputStream.java +++ b/java/src/main/java/com/google/protobuf/CodedInputStream.java @@ -373,14 +373,14 @@ public final class CodedInputStream { if (size <= (bufferSize - bufferPos) && size > 0) { // Fast path: We already have the bytes in a contiguous buffer, so // just copy directly from it. - final String result = new String(buffer, bufferPos, size, "UTF-8"); + final String result = new String(buffer, bufferPos, size, Internal.UTF_8); bufferPos += size; return result; } else if (size == 0) { return ""; } else { // Slow path: Build a byte array first then copy it. - return new String(readRawBytesSlowPath(size), "UTF-8"); + return new String(readRawBytesSlowPath(size), Internal.UTF_8); } } @@ -409,7 +409,7 @@ public final class CodedInputStream { if (!Utf8.isValidUtf8(bytes, pos, pos + size)) { throw InvalidProtocolBufferException.invalidUtf8(); } - return new String(bytes, pos, size, "UTF-8"); + return new String(bytes, pos, size, Internal.UTF_8); } /** Read a {@code group} field value from the stream. */ diff --git a/java/src/main/java/com/google/protobuf/CodedOutputStream.java b/java/src/main/java/com/google/protobuf/CodedOutputStream.java index fafe0356..954fde08 100644 --- a/java/src/main/java/com/google/protobuf/CodedOutputStream.java +++ b/java/src/main/java/com/google/protobuf/CodedOutputStream.java @@ -32,7 +32,6 @@ package com.google.protobuf; import java.io.IOException; import java.io.OutputStream; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; /** @@ -144,7 +143,7 @@ public final class CodedOutputStream { int bufferSize) { return newInstance(new ByteBufferOutputStream(byteBuffer), bufferSize); } - + private static class ByteBufferOutputStream extends OutputStream { private final ByteBuffer byteBuffer; public ByteBufferOutputStream(ByteBuffer byteBuffer) { @@ -420,7 +419,7 @@ public final class CodedOutputStream { // Unfortunately there does not appear to be any way to tell Java to encode // UTF-8 directly into our buffer, so we have to let it create its own byte // array and then copy. - final byte[] bytes = value.getBytes("UTF-8"); + final byte[] bytes = value.getBytes(Internal.UTF_8); writeRawVarint32(bytes.length); writeRawBytes(bytes); } @@ -750,7 +749,7 @@ public final class CodedOutputStream { computeUInt32Size(WireFormat.MESSAGE_SET_TYPE_ID, fieldNumber) + computeLazyFieldSize(WireFormat.MESSAGE_SET_MESSAGE, value); } - + // ----------------------------------------------------------------- /** @@ -827,13 +826,9 @@ public final class CodedOutputStream { * {@code string} field. */ public static int computeStringSizeNoTag(final String value) { - try { - final byte[] bytes = value.getBytes("UTF-8"); - return computeRawVarint32Size(bytes.length) + - bytes.length; - } catch (UnsupportedEncodingException e) { - throw new RuntimeException("UTF-8 not supported.", e); - } + final byte[] bytes = value.getBytes(Internal.UTF_8); + return computeRawVarint32Size(bytes.length) + + bytes.length; } /** diff --git a/java/src/main/java/com/google/protobuf/Descriptors.java b/java/src/main/java/com/google/protobuf/Descriptors.java index 806f46c1..b60bd36b 100644 --- a/java/src/main/java/com/google/protobuf/Descriptors.java +++ b/java/src/main/java/com/google/protobuf/Descriptors.java @@ -43,7 +43,6 @@ import java.util.Map; import java.util.Set; import java.util.WeakHashMap; import java.util.logging.Logger; -import java.io.UnsupportedEncodingException; /** * Contains a collection of classes which describe protocol message types. @@ -256,7 +255,7 @@ public final class Descriptors { throws DescriptorValidationException { return buildFrom(proto, dependencies, false); } - + /** * Construct a {@code FileDescriptor}. @@ -319,12 +318,7 @@ public final class Descriptors { } final byte[] descriptorBytes; - try { - descriptorBytes = descriptorData.toString().getBytes("ISO-8859-1"); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException( - "Standard encoding ISO-8859-1 not supported by JVM.", e); - } + descriptorBytes = descriptorData.toString().getBytes(Internal.ISO_8859_1); FileDescriptorProto proto; try { @@ -495,9 +489,9 @@ public final class Descriptors { proto.getExtension(i), this, null, i, true); } } - + /** - * Create a placeholder FileDescriptor for a message Descriptor. + * Create a placeholder FileDescriptor for a message Descriptor. */ FileDescriptor(String packageName, Descriptor message) throws DescriptorValidationException { @@ -561,7 +555,7 @@ public final class Descriptors { extensions[i].setProto(proto.getExtension(i)); } } - + boolean supportsUnknownEnumValue() { return getSyntax() == Syntax.PROTO3; } @@ -745,7 +739,7 @@ public final class Descriptors { this.fields = new FieldDescriptor[0]; this.extensions = new FieldDescriptor[0]; this.oneofs = new OneofDescriptor[0]; - + // Create a placeholder FileDescriptor to hold this message. this.file = new FileDescriptor(packageName, this); } @@ -1333,8 +1327,8 @@ public final class Descriptors { "Message type had default value."); } } catch (NumberFormatException e) { - throw new DescriptorValidationException(this, - "Could not parse default value: \"" + + throw new DescriptorValidationException(this, + "Could not parse default value: \"" + proto.getDefaultValue() + '\"', e); } } else { @@ -1456,7 +1450,7 @@ public final class Descriptors { return file.pool.enumValuesByNumber.get( new DescriptorPool.DescriptorIntPair(this, number)); } - + /** * Get the enum value for a number. If no enum value has this number, * construct an EnumValueDescriptor for it. @@ -1494,7 +1488,7 @@ public final class Descriptors { // then remove the whole entry. This way unknown descriptors will // be freed automatically and we don't need to do anything to // clean-up unused map entries. - + // Note: We must use "new Integer(number)" here because we don't want // these Integer objects to be cached. Integer key = new Integer(number); @@ -1509,7 +1503,7 @@ public final class Descriptors { } return result; } - + // Used in tests only. int getUnknownEnumValueDescriptorCount() { return unknownValues.size(); @@ -1585,7 +1579,7 @@ public final class Descriptors { /** Get the value's number. */ public int getNumber() { return proto.getNumber(); } - + @Override public String toString() { return proto.getName(); } @@ -1627,7 +1621,7 @@ public final class Descriptors { file.pool.addSymbol(this); file.pool.addEnumValueByNumber(this); } - + private Integer number; // Create an unknown enum value. private EnumValueDescriptor( @@ -1643,7 +1637,7 @@ public final class Descriptors { this.type = parent; this.fullName = parent.getFullName() + '.' + proto.getName(); this.number = number; - + // Don't add this descriptor into pool. } @@ -1930,13 +1924,13 @@ public final class Descriptors { * descriptors defined in a particular file. */ private static final class DescriptorPool { - - /** Defines what subclass of descriptors to search in the descriptor pool. + + /** Defines what subclass of descriptors to search in the descriptor pool. */ enum SearchFilter { TYPES_ONLY, AGGREGATES_ONLY, ALL_SYMBOLS } - + DescriptorPool(final FileDescriptor[] dependencies, boolean allowUnknownDependencies) { this.dependencies = new HashSet<FileDescriptor>(); @@ -1982,9 +1976,9 @@ public final class Descriptors { GenericDescriptor findSymbol(final String fullName) { return findSymbol(fullName, SearchFilter.ALL_SYMBOLS); } - - /** Find a descriptor by fully-qualified name and given option to only - * search valid field type descriptors. + + /** Find a descriptor by fully-qualified name and given option to only + * search valid field type descriptors. */ GenericDescriptor findSymbol(final String fullName, final SearchFilter filter) { @@ -2013,18 +2007,18 @@ public final class Descriptors { /** Checks if the descriptor is a valid type for a message field. */ boolean isType(GenericDescriptor descriptor) { - return (descriptor instanceof Descriptor) || + return (descriptor instanceof Descriptor) || (descriptor instanceof EnumDescriptor); } - + /** Checks if the descriptor is a valid namespace type. */ boolean isAggregate(GenericDescriptor descriptor) { - return (descriptor instanceof Descriptor) || - (descriptor instanceof EnumDescriptor) || - (descriptor instanceof PackageDescriptor) || + return (descriptor instanceof Descriptor) || + (descriptor instanceof EnumDescriptor) || + (descriptor instanceof PackageDescriptor) || (descriptor instanceof ServiceDescriptor); } - + /** * Look up a type descriptor by name, relative to some other descriptor. * The name may be fully-qualified (with a leading '.'), @@ -2082,7 +2076,7 @@ public final class Descriptors { // Append firstPart and try to find scopeToTry.append(firstPart); - result = findSymbol(scopeToTry.toString(), + result = findSymbol(scopeToTry.toString(), DescriptorPool.SearchFilter.AGGREGATES_ONLY); if (result != null) { diff --git a/java/src/main/java/com/google/protobuf/Internal.java b/java/src/main/java/com/google/protobuf/Internal.java index 5a0de6d1..74bf44c0 100644 --- a/java/src/main/java/com/google/protobuf/Internal.java +++ b/java/src/main/java/com/google/protobuf/Internal.java @@ -30,9 +30,8 @@ package com.google.protobuf; -import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; +import java.nio.charset.Charset; import java.util.AbstractList; import java.util.AbstractMap; import java.util.AbstractSet; @@ -51,6 +50,10 @@ import java.util.Set; * @author kenton@google.com (Kenton Varda) */ public class Internal { + + protected static final Charset UTF_8 = Charset.forName("UTF-8"); + protected static final Charset ISO_8859_1 = Charset.forName("ISO-8859-1"); + /** * Helper called by generated code to construct default values for string * fields. @@ -80,14 +83,7 @@ public class Internal { * generated code calls this automatically. */ public static String stringDefaultValue(String bytes) { - try { - return new String(bytes.getBytes("ISO-8859-1"), "UTF-8"); - } catch (UnsupportedEncodingException e) { - // This should never happen since all JVMs are required to implement - // both of the above character sets. - throw new IllegalStateException( - "Java VM does not support a standard character set.", e); - } + return new String(bytes.getBytes(ISO_8859_1), UTF_8); } /** @@ -99,37 +95,23 @@ public class Internal { * embed raw bytes as a string literal with ISO-8859-1 encoding. */ public static ByteString bytesDefaultValue(String bytes) { - try { - return ByteString.copyFrom(bytes.getBytes("ISO-8859-1")); - } catch (UnsupportedEncodingException e) { - // This should never happen since all JVMs are required to implement - // ISO-8859-1. - throw new IllegalStateException( - "Java VM does not support a standard character set.", e); - } + return ByteString.copyFrom(bytes.getBytes(ISO_8859_1)); } /** * Helper called by generated code to construct default values for bytes * fields. * <p> - * This is like {@link #bytesDefaultValue}, but returns a byte array. + * This is like {@link #bytesDefaultValue}, but returns a byte array. */ public static byte[] byteArrayDefaultValue(String bytes) { - try { - return bytes.getBytes("ISO-8859-1"); - } catch (UnsupportedEncodingException e) { - // This should never happen since all JVMs are required to implement - // ISO-8859-1. - throw new IllegalStateException( - "Java VM does not support a standard character set.", e); - } + return bytes.getBytes(ISO_8859_1); } /** * Helper called by generated code to construct default values for bytes * fields. * <p> - * This is like {@link #bytesDefaultValue}, but returns a ByteBuffer. + * This is like {@link #bytesDefaultValue}, but returns a ByteBuffer. */ public static ByteBuffer byteBufferDefaultValue(String bytes) { return ByteBuffer.wrap(byteArrayDefaultValue(bytes)); @@ -185,7 +167,7 @@ public class Internal { public static boolean isValidUtf8(ByteString byteString) { return byteString.isValidUtf8(); } - + /** * Like {@link #isValidUtf8(ByteString)} but for byte arrays. */ @@ -197,22 +179,14 @@ public class Internal { * Helper method to get the UTF-8 bytes of a string. */ public static byte[] toByteArray(String value) { - try { - return value.getBytes("UTF-8"); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException("UTF-8 not supported?", e); - } + return value.getBytes(UTF_8); } - + /** * Helper method to convert a byte array to a string using UTF-8 encoding. */ public static String toStringUtf8(byte[] bytes) { - try { - return new String(bytes, "UTF-8"); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException("UTF-8 not supported?", e); - } + return new String(bytes, UTF_8); } /** @@ -274,7 +248,7 @@ public class Internal { } return hash; } - + /** * Helper method for implementing {@link Message#equals(Object)} for bytes field. */ @@ -298,7 +272,7 @@ public class Internal { } return hash; } - + /** * Helper method for implementing {@link Message#hashCode()} for bytes field. */ @@ -309,7 +283,7 @@ public class Internal { // based hashCode() method. return LiteralByteString.hashCode(bytes); } - + /** * Helper method for implementing {@link Message#equals(Object)} for bytes * field. @@ -322,7 +296,7 @@ public class Internal { // compare all the content. return a.duplicate().clear().equals(b.duplicate().clear()); } - + /** * Helper method for implementing {@link Message#equals(Object)} for bytes * field. @@ -351,9 +325,9 @@ public class Internal { } return hash; } - + private static final int DEFAULT_BUFFER_SIZE = 4096; - + /** * Helper method for implementing {@link Message#hashCode()} for bytes * field. @@ -382,18 +356,18 @@ public class Internal { return h == 0 ? 1 : h; } } - + /** * An empty byte array constant used in generated code. */ public static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; - + /** * An empty byte array constant used in generated code. */ public static final ByteBuffer EMPTY_BYTE_BUFFER = ByteBuffer.wrap(EMPTY_BYTE_ARRAY); - + /** An empty coded input stream constant used in generated code. */ public static final CodedInputStream EMPTY_CODED_INPUT_STREAM = CodedInputStream.newInstance(EMPTY_BYTE_ARRAY); @@ -458,13 +432,13 @@ public class Internal { private final Map<K, RealValue> realMap; private final Converter<RealValue, V> valueConverter; - + public MapAdapter(Map<K, RealValue> realMap, Converter<RealValue, V> valueConverter) { this.realMap = realMap; this.valueConverter = valueConverter; } - + @SuppressWarnings("unchecked") @Override public V get(Object key) { @@ -474,7 +448,7 @@ public class Internal { } return valueConverter.doForward(result); } - + @Override public V put(K key, V value) { RealValue oldValue = realMap.put(key, valueConverter.doBackward(value)); @@ -488,13 +462,13 @@ public class Internal { public Set<java.util.Map.Entry<K, V>> entrySet() { return new SetAdapter(realMap.entrySet()); } - + private class SetAdapter extends AbstractSet<Map.Entry<K, V>> { private final Set<Map.Entry<K, RealValue>> realSet; public SetAdapter(Set<Map.Entry<K, RealValue>> realSet) { this.realSet = realSet; } - + @Override public Iterator<java.util.Map.Entry<K, V>> iterator() { return new IteratorAdapter(realSet.iterator()); @@ -503,17 +477,17 @@ public class Internal { @Override public int size() { return realSet.size(); - } + } } - + private class IteratorAdapter implements Iterator<Map.Entry<K, V>> { private final Iterator<Map.Entry<K, RealValue>> realIterator; - + public IteratorAdapter( Iterator<Map.Entry<K, RealValue>> realIterator) { this.realIterator = realIterator; } - + @Override public boolean hasNext() { return realIterator.hasNext(); @@ -529,14 +503,14 @@ public class Internal { realIterator.remove(); } } - + private class EntryAdapter implements Map.Entry<K, V> { private final Map.Entry<K, RealValue> realEntry; - + public EntryAdapter(Map.Entry<K, RealValue> realEntry) { this.realEntry = realEntry; } - + @Override public K getKey() { return realEntry.getKey(); diff --git a/java/src/main/java/com/google/protobuf/LiteralByteString.java b/java/src/main/java/com/google/protobuf/LiteralByteString.java index 6893ddf1..c5a8512a 100644 --- a/java/src/main/java/com/google/protobuf/LiteralByteString.java +++ b/java/src/main/java/com/google/protobuf/LiteralByteString.java @@ -34,8 +34,8 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; import java.util.NoSuchElementException; @@ -112,7 +112,7 @@ class LiteralByteString extends ByteString { // ByteString -> byte[] @Override - protected void copyToInternal(byte[] target, int sourceOffset, + protected void copyToInternal(byte[] target, int sourceOffset, int targetOffset, int numberToCopy) { // Optimized form, not for subclasses, since we don't call // getOffsetIntoBytes() or check the 'numberToCopy' parameter. @@ -152,13 +152,8 @@ class LiteralByteString extends ByteString { } @Override - public String toString(String charsetName) - throws UnsupportedEncodingException { - // Optimize for empty strings, but ensure we don't silently ignore invalid - // encodings. - return size() == 0 && UTF_8.equals(charsetName) - ? "" - : new String(bytes, getOffsetIntoBytes(), size(), charsetName); + protected String toStringInternal(Charset charset) { + return new String(bytes, getOffsetIntoBytes(), size(), charset); } // ================================================================= @@ -199,12 +194,12 @@ class LiteralByteString extends ByteString { LiteralByteString otherAsLiteral = (LiteralByteString) other; // If we know the hash codes and they are not equal, we know the byte // strings are not equal. - if (hash != 0 - && otherAsLiteral.hash != 0 + if (hash != 0 + && otherAsLiteral.hash != 0 && hash != otherAsLiteral.hash) { return false; } - + return equalsRange((LiteralByteString) other, 0, size()); } else if (other instanceof RopeByteString) { return other.equals(this); @@ -285,14 +280,14 @@ class LiteralByteString extends ByteString { protected int partialHash(int h, int offset, int length) { return hashCode(h, bytes, getOffsetIntoBytes() + offset, length); } - + static int hashCode(int h, byte[] bytes, int offset, int length) { for (int i = offset; i < offset + length; i++) { h = h * 31 + bytes[i]; } return h; } - + static int hashCode(byte[] bytes) { int h = hashCode(bytes.length, bytes, 0, bytes.length); return h == 0 ? 1 : h; diff --git a/java/src/main/java/com/google/protobuf/RopeByteString.java b/java/src/main/java/com/google/protobuf/RopeByteString.java index 168bcce2..4f3eb127 100644 --- a/java/src/main/java/com/google/protobuf/RopeByteString.java +++ b/java/src/main/java/com/google/protobuf/RopeByteString.java @@ -30,14 +30,14 @@ package com.google.protobuf; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InvalidObjectException; import java.io.ObjectInputStream; import java.io.OutputStream; -import java.io.UnsupportedEncodingException; -import java.io.ByteArrayInputStream; import java.nio.ByteBuffer; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -418,13 +418,8 @@ class RopeByteString extends ByteString { } @Override - public String toString(String charsetName) - throws UnsupportedEncodingException { - // Optimize for empty strings, but ensure we don't silently ignore invalid - // encodings. - return size() == 0 && UTF_8.equals(charsetName) - ? "" - : new String(toByteArray(), charsetName); + protected String toStringInternal(Charset charset) { + return new String(toByteArray(), charset); } // ================================================================= diff --git a/java/src/main/java/com/google/protobuf/Utf8.java b/java/src/main/java/com/google/protobuf/Utf8.java index 4d0ef53b..4271b41b 100644 --- a/java/src/main/java/com/google/protobuf/Utf8.java +++ b/java/src/main/java/com/google/protobuf/Utf8.java @@ -46,7 +46,7 @@ package com.google.protobuf; * <p>The byte sequences considered valid by this class are exactly * those that can be roundtrip converted to Strings and back to bytes * using the UTF-8 charset, without loss: <pre> {@code - * Arrays.equals(bytes, new String(bytes, "UTF-8").getBytes("UTF-8")) + * Arrays.equals(bytes, new String(bytes, Internal.UTF_8).getBytes(Internal.UTF_8)) * }</pre> * * <p>See the Unicode Standard,</br> diff --git a/java/src/test/java/com/google/protobuf/BoundedByteStringTest.java b/java/src/test/java/com/google/protobuf/BoundedByteStringTest.java index 6c9596ca..1562a1a6 100644 --- a/java/src/test/java/com/google/protobuf/BoundedByteStringTest.java +++ b/java/src/test/java/com/google/protobuf/BoundedByteStringTest.java @@ -62,7 +62,7 @@ public class BoundedByteStringTest extends LiteralByteStringTest { @Override public void testToString() throws UnsupportedEncodingException { String testString = "I love unicode \u1234\u5678 characters"; - LiteralByteString unicode = new LiteralByteString(testString.getBytes(UTF_8)); + LiteralByteString unicode = new LiteralByteString(testString.getBytes(Internal.UTF_8)); ByteString chopped = unicode.substring(2, unicode.size() - 6); assertEquals(classUnderTest + ".substring() must have the expected type", classUnderTest, getActualClassName(chopped)); @@ -72,6 +72,19 @@ public class BoundedByteStringTest extends LiteralByteStringTest { testString.substring(2, testString.length() - 6), roundTripString); } + @Override + public void testCharsetToString() throws UnsupportedEncodingException { + String testString = "I love unicode \u1234\u5678 characters"; + LiteralByteString unicode = new LiteralByteString(testString.getBytes(Internal.UTF_8)); + ByteString chopped = unicode.substring(2, unicode.size() - 6); + assertEquals(classUnderTest + ".substring() must have the expected type", + classUnderTest, getActualClassName(chopped)); + + String roundTripString = chopped.toString(Internal.UTF_8); + assertEquals(classUnderTest + " unicode bytes must match", + testString.substring(2, testString.length() - 6), roundTripString); + } + public void testJavaSerialization() throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); ObjectOutputStream oos = new ObjectOutputStream(out); diff --git a/java/src/test/java/com/google/protobuf/ByteStringTest.java b/java/src/test/java/com/google/protobuf/ByteStringTest.java index 15de17cd..8af5dcde 100644 --- a/java/src/test/java/com/google/protobuf/ByteStringTest.java +++ b/java/src/test/java/com/google/protobuf/ByteStringTest.java @@ -41,6 +41,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -56,7 +57,7 @@ import java.util.Random; */ public class ByteStringTest extends TestCase { - private static final String UTF_16 = "UTF-16"; + private static final Charset UTF_16 = Charset.forName("UTF-16"); static byte[] getTestBytes(int size, long seed) { Random random = new Random(seed); @@ -139,7 +140,7 @@ public class ByteStringTest extends TestCase { public void testCopyFrom_Utf8() throws UnsupportedEncodingException { String testString = "I love unicode \u1234\u5678 characters"; ByteString byteString = ByteString.copyFromUtf8(testString); - byte[] testBytes = testString.getBytes("UTF-8"); + byte[] testBytes = testString.getBytes(Internal.UTF_8); assertTrue("copyFromUtf8 string must respect the charset", isArrayRange(byteString.toByteArray(), testBytes, 0, testBytes.length)); } @@ -380,7 +381,7 @@ public class ByteStringTest extends TestCase { return -1; } } - + // A stream which exposes the byte array passed into write(byte[], int, int). private static class EvilOutputStream extends OutputStream { public byte[] capturedArray = null; @@ -400,7 +401,7 @@ public class ByteStringTest extends TestCase { public void testToStringUtf8() throws UnsupportedEncodingException { String testString = "I love unicode \u1234\u5678 characters"; - byte[] testBytes = testString.getBytes("UTF-8"); + byte[] testBytes = testString.getBytes(Internal.UTF_8); ByteString byteString = ByteString.copyFrom(testBytes); assertEquals("copyToStringUtf8 must respect the charset", testString, byteString.toStringUtf8()); @@ -492,13 +493,13 @@ public class ByteStringTest extends TestCase { isArrayRange(bytes, byteString.toByteArray(), 0, bytes.length)); } } - + public void testNewOutputEmpty() throws IOException { // Make sure newOutput() correctly builds empty byte strings ByteString byteString = ByteString.newOutput().toByteString(); assertEquals(ByteString.EMPTY, byteString); } - + public void testNewOutput_Mutating() throws IOException { Output os = ByteString.newOutput(5); os.write(new byte[] {1, 2, 3, 4, 5}); @@ -705,14 +706,14 @@ public class ByteStringTest extends TestCase { } return pieces; } - + private byte[] substringUsingWriteTo( ByteString data, int offset, int length) throws IOException { ByteArrayOutputStream output = new ByteArrayOutputStream(); data.writeTo(output, offset, length); return output.toByteArray(); } - + public void testWriteToOutputStream() throws Exception { // Choose a size large enough so when two ByteStrings are concatenated they // won't be merged into one byte array due to some optimizations. @@ -727,7 +728,7 @@ public class ByteStringTest extends TestCase { byte[] result = substringUsingWriteTo(left, 1, 1); assertEquals(1, result.length); assertEquals((byte) 11, result[0]); - + byte[] data2 = new byte[dataSize]; for (int i = 0; i < data1.length; i++) { data2[i] = (byte) 2; diff --git a/java/src/test/java/com/google/protobuf/CodedOutputStreamTest.java b/java/src/test/java/com/google/protobuf/CodedOutputStreamTest.java index 0785cc91..365789c0 100644 --- a/java/src/test/java/com/google/protobuf/CodedOutputStreamTest.java +++ b/java/src/test/java/com/google/protobuf/CodedOutputStreamTest.java @@ -321,7 +321,7 @@ public class CodedOutputStreamTest extends TestCase { final int BUFFER_SIZE = 4 * 1024; ByteArrayOutputStream outputStream = new ByteArrayOutputStream(BUFFER_SIZE); CodedOutputStream codedStream = CodedOutputStream.newInstance(outputStream); - byte[] value = "abcde".getBytes("UTF-8"); + byte[] value = "abcde".getBytes(Internal.UTF_8); for (int i = 0; i < 1024; ++i) { codedStream.writeRawBytes(value, 0, value.length); } @@ -330,7 +330,7 @@ public class CodedOutputStreamTest extends TestCase { assertTrue(codedStream.getTotalBytesWritten() > BUFFER_SIZE); assertEquals(value.length * 1024, codedStream.getTotalBytesWritten()); } - + public void testWriteToByteBuffer() throws Exception { final int bufferSize = 16 * 1024; ByteBuffer buffer = ByteBuffer.allocate(bufferSize); @@ -351,7 +351,7 @@ public class CodedOutputStreamTest extends TestCase { codedStream.writeRawByte((byte) 3); } codedStream.flush(); - + // Check that data is correctly written to the ByteBuffer. assertEquals(0, buffer.remaining()); buffer.flip(); @@ -365,9 +365,9 @@ public class CodedOutputStreamTest extends TestCase { assertEquals((byte) 3, buffer.get()); } } - + public void testWriteByteBuffer() throws Exception { - byte[] value = "abcde".getBytes("UTF-8"); + byte[] value = "abcde".getBytes(Internal.UTF_8); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); CodedOutputStream codedStream = CodedOutputStream.newInstance(outputStream); ByteBuffer byteBuffer = ByteBuffer.wrap(value, 0, 1); @@ -377,10 +377,10 @@ public class CodedOutputStreamTest extends TestCase { // The above call shouldn't affect the ByteBuffer's state. assertEquals(0, byteBuffer.position()); assertEquals(1, byteBuffer.limit()); - + // The correct way to write part of an array using ByteBuffer. codedStream.writeRawBytes(ByteBuffer.wrap(value, 2, 1).slice()); - + codedStream.flush(); byte[] result = outputStream.toByteArray(); assertEquals(6, result.length); diff --git a/java/src/test/java/com/google/protobuf/DescriptorsTest.java b/java/src/test/java/com/google/protobuf/DescriptorsTest.java index 05832a12..6cfa18d5 100644 --- a/java/src/test/java/com/google/protobuf/DescriptorsTest.java +++ b/java/src/test/java/com/google/protobuf/DescriptorsTest.java @@ -35,32 +35,28 @@ import com.google.protobuf.DescriptorProtos.EnumDescriptorProto; import com.google.protobuf.DescriptorProtos.EnumValueDescriptorProto; import com.google.protobuf.DescriptorProtos.FieldDescriptorProto; import com.google.protobuf.DescriptorProtos.FileDescriptorProto; -import com.google.protobuf.Descriptors.DescriptorValidationException; -import com.google.protobuf.Descriptors.FileDescriptor; import com.google.protobuf.Descriptors.Descriptor; -import com.google.protobuf.Descriptors.FieldDescriptor; -import com.google.protobuf.Descriptors.OneofDescriptor; +import com.google.protobuf.Descriptors.DescriptorValidationException; import com.google.protobuf.Descriptors.EnumDescriptor; import com.google.protobuf.Descriptors.EnumValueDescriptor; -import com.google.protobuf.Descriptors.ServiceDescriptor; +import com.google.protobuf.Descriptors.FieldDescriptor; +import com.google.protobuf.Descriptors.FileDescriptor; import com.google.protobuf.Descriptors.MethodDescriptor; - +import com.google.protobuf.Descriptors.OneofDescriptor; +import com.google.protobuf.Descriptors.ServiceDescriptor; import com.google.protobuf.test.UnittestImport; import com.google.protobuf.test.UnittestImport.ImportEnum; -import com.google.protobuf.test.UnittestImport.ImportMessage; +import protobuf_unittest.TestCustomOptions; +import protobuf_unittest.UnittestCustomOptions; import protobuf_unittest.UnittestProto; import protobuf_unittest.UnittestProto.ForeignEnum; import protobuf_unittest.UnittestProto.ForeignMessage; -import protobuf_unittest.UnittestProto.TestAllTypes; import protobuf_unittest.UnittestProto.TestAllExtensions; +import protobuf_unittest.UnittestProto.TestAllTypes; import protobuf_unittest.UnittestProto.TestExtremeDefaultValues; import protobuf_unittest.UnittestProto.TestMultipleExtensionRanges; import protobuf_unittest.UnittestProto.TestRequired; import protobuf_unittest.UnittestProto.TestService; -import protobuf_unittest.UnittestCustomOptions; - -import protobuf_unittest.TestCustomOptions; - import junit.framework.TestCase; @@ -286,7 +282,7 @@ public class DescriptorsTest extends TestCase { d = TestExtremeDefaultValues.getDescriptor(); assertEquals( ByteString.copyFrom( - "\0\001\007\b\f\n\r\t\013\\\'\"\u00fe".getBytes("ISO-8859-1")), + "\0\001\007\b\f\n\r\t\013\\\'\"\u00fe".getBytes(Internal.ISO_8859_1)), d.findFieldByName("escaped_bytes").getDefaultValue()); assertEquals(-1, d.findFieldByName("large_uint32").getDefaultValue()); assertEquals(-1L, d.findFieldByName("large_uint64").getDefaultValue()); @@ -497,7 +493,7 @@ public class DescriptorsTest extends TestCase { .build(); // translate and crosslink FileDescriptor file = - Descriptors.FileDescriptor.buildFrom(fileDescriptorProto, + Descriptors.FileDescriptor.buildFrom(fileDescriptorProto, new FileDescriptor[0]); // verify resulting descriptors assertNotNull(file); @@ -518,7 +514,7 @@ public class DescriptorsTest extends TestCase { } assertTrue(barFound); } - + public void testDependencyOrder() throws Exception { FileDescriptorProto fooProto = FileDescriptorProto.newBuilder() .setName("foo.proto").build(); @@ -537,14 +533,14 @@ public class DescriptorsTest extends TestCase { new FileDescriptor[0]); FileDescriptor barFile = Descriptors.FileDescriptor.buildFrom(barProto, new FileDescriptor[] {fooFile}); - - // Items in the FileDescriptor array can be in any order. + + // Items in the FileDescriptor array can be in any order. Descriptors.FileDescriptor.buildFrom(bazProto, new FileDescriptor[] {fooFile, barFile}); Descriptors.FileDescriptor.buildFrom(bazProto, new FileDescriptor[] {barFile, fooFile}); } - + public void testInvalidPublicDependency() throws Exception { FileDescriptorProto fooProto = FileDescriptorProto.newBuilder() .setName("foo.proto").build(); @@ -628,7 +624,7 @@ public class DescriptorsTest extends TestCase { Descriptors.FileDescriptor.buildFrom( fooProto, new FileDescriptor[] {forwardFile}); } - + /** * Tests the translate/crosslink for an example with a more complex namespace * referencing. @@ -677,7 +673,7 @@ public class DescriptorsTest extends TestCase { assertTrue(field.getEnumType().getFile().getName().equals("bar.proto")); assertTrue(field.getEnumType().getFile().getPackage().equals( "a.b.c.d.bar.shared")); - } + } } public void testOneofDescriptor() throws Exception { diff --git a/java/src/test/java/com/google/protobuf/IsValidUtf8Test.java b/java/src/test/java/com/google/protobuf/IsValidUtf8Test.java index 89b64519..8751baae 100644 --- a/java/src/test/java/com/google/protobuf/IsValidUtf8Test.java +++ b/java/src/test/java/com/google/protobuf/IsValidUtf8Test.java @@ -72,8 +72,11 @@ public class IsValidUtf8Test extends TestCase { * Tests that round tripping of all three byte permutations work. */ public void testIsValidUtf8_3Bytes() throws UnsupportedEncodingException { - IsValidUtf8TestUtil.testBytes(3, - IsValidUtf8TestUtil.EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT); + // Travis' OOM killer doesn't like this test + if (System.getenv("TRAVIS") == null) { + IsValidUtf8TestUtil.testBytes(3, + IsValidUtf8TestUtil.EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT); + } } /** diff --git a/java/src/test/java/com/google/protobuf/IsValidUtf8TestUtil.java b/java/src/test/java/com/google/protobuf/IsValidUtf8TestUtil.java index f41595ec..321669f3 100644 --- a/java/src/test/java/com/google/protobuf/IsValidUtf8TestUtil.java +++ b/java/src/test/java/com/google/protobuf/IsValidUtf8TestUtil.java @@ -33,18 +33,17 @@ package com.google.protobuf; import static junit.framework.Assert.*; import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CoderResult; +import java.nio.charset.CodingErrorAction; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Random; import java.util.logging.Logger; -import java.nio.charset.CharsetDecoder; -import java.nio.charset.Charset; -import java.nio.charset.CodingErrorAction; -import java.nio.charset.CharsetEncoder; -import java.nio.charset.CoderResult; -import java.nio.ByteBuffer; -import java.nio.CharBuffer; /** * Shared testing code for {@link IsValidUtf8Test} and @@ -220,8 +219,8 @@ class IsValidUtf8TestUtil { } ByteString bs = ByteString.copyFrom(bytes); boolean isRoundTrippable = bs.isValidUtf8(); - String s = new String(bytes, "UTF-8"); - byte[] bytesReencoded = s.getBytes("UTF-8"); + String s = new String(bytes, Internal.UTF_8); + byte[] bytesReencoded = s.getBytes(Internal.UTF_8); boolean bytesEqual = Arrays.equals(bytes, bytesReencoded); if (bytesEqual != isRoundTrippable) { @@ -313,10 +312,10 @@ class IsValidUtf8TestUtil { void testBytesUsingByteBuffers( int numBytes, long expectedCount, long start, long lim) throws UnsupportedEncodingException { - CharsetDecoder decoder = Charset.forName("UTF-8").newDecoder() + CharsetDecoder decoder = Internal.UTF_8.newDecoder() .onMalformedInput(CodingErrorAction.REPLACE) .onUnmappableCharacter(CodingErrorAction.REPLACE); - CharsetEncoder encoder = Charset.forName("UTF-8").newEncoder() + CharsetEncoder encoder = Internal.UTF_8.newEncoder() .onMalformedInput(CodingErrorAction.REPLACE) .onUnmappableCharacter(CodingErrorAction.REPLACE); byte[] bytes = new byte[numBytes]; diff --git a/java/src/test/java/com/google/protobuf/LiteralByteStringTest.java b/java/src/test/java/com/google/protobuf/LiteralByteStringTest.java index 046832de..7b201a9d 100644 --- a/java/src/test/java/com/google/protobuf/LiteralByteStringTest.java +++ b/java/src/test/java/com/google/protobuf/LiteralByteStringTest.java @@ -248,7 +248,7 @@ public class LiteralByteStringTest extends TestCase { assertTrue(classUnderTest + ".writeTo() must give back the same bytes", Arrays.equals(referenceBytes, roundTripBytes)); } - + public void testWriteTo_mutating() throws IOException { OutputStream os = new OutputStream() { @Override @@ -293,14 +293,21 @@ public class LiteralByteStringTest extends TestCase { public void testToString() throws UnsupportedEncodingException { String testString = "I love unicode \u1234\u5678 characters"; - LiteralByteString unicode = new LiteralByteString(testString.getBytes(UTF_8)); + LiteralByteString unicode = new LiteralByteString(testString.getBytes(Internal.UTF_8)); String roundTripString = unicode.toString(UTF_8); assertEquals(classUnderTest + " unicode must match", testString, roundTripString); } + public void testCharsetToString() throws UnsupportedEncodingException { + String testString = "I love unicode \u1234\u5678 characters"; + LiteralByteString unicode = new LiteralByteString(testString.getBytes(Internal.UTF_8)); + String roundTripString = unicode.toString(Internal.UTF_8); + assertEquals(classUnderTest + " unicode must match", testString, roundTripString); + } + public void testToString_returnsCanonicalEmptyString() throws UnsupportedEncodingException{ assertSame(classUnderTest + " must be the same string references", - ByteString.EMPTY.toString(UTF_8), new LiteralByteString(new byte[]{}).toString(UTF_8)); + ByteString.EMPTY.toString(Internal.UTF_8), new LiteralByteString(new byte[]{}).toString(Internal.UTF_8)); } public void testToString_raisesException() throws UnsupportedEncodingException{ diff --git a/java/src/test/java/com/google/protobuf/RopeByteStringSubstringTest.java b/java/src/test/java/com/google/protobuf/RopeByteStringSubstringTest.java index 212f8d68..cc385599 100644 --- a/java/src/test/java/com/google/protobuf/RopeByteStringSubstringTest.java +++ b/java/src/test/java/com/google/protobuf/RopeByteStringSubstringTest.java @@ -35,7 +35,7 @@ import java.util.Iterator; /** * This class tests {@link RopeByteString#substring(int, int)} by inheriting the tests from - * {@link LiteralByteStringTest}. Only a couple of methods are overridden. + * {@link LiteralByteStringTest}. Only a couple of methods are overridden. * * @author carlanton@google.com (Carl Haverl) */ @@ -94,4 +94,34 @@ public class RopeByteStringSubstringTest extends LiteralByteStringTest { assertEquals(classUnderTest + " string must must have same hashCode as the flat string", flatString.hashCode(), unicode.hashCode()); } + + @Override + public void testCharsetToString() throws UnsupportedEncodingException { + String sourceString = "I love unicode \u1234\u5678 characters"; + ByteString sourceByteString = ByteString.copyFromUtf8(sourceString); + int copies = 250; + + // By building the RopeByteString by concatenating, this is actually a fairly strenuous test. + StringBuilder builder = new StringBuilder(copies * sourceString.length()); + ByteString unicode = ByteString.EMPTY; + for (int i = 0; i < copies; ++i) { + builder.append(sourceString); + unicode = RopeByteString.concatenate(unicode, sourceByteString); + } + String testString = builder.toString(); + + // Do the substring part + testString = testString.substring(2, testString.length() - 6); + unicode = unicode.substring(2, unicode.size() - 6); + + assertEquals(classUnderTest + " from string must have the expected type", + classUnderTest, getActualClassName(unicode)); + String roundTripString = unicode.toString(Internal.UTF_8); + assertEquals(classUnderTest + " unicode bytes must match", + testString, roundTripString); + ByteString flatString = ByteString.copyFromUtf8(testString); + assertEquals(classUnderTest + " string must equal the flat string", flatString, unicode); + assertEquals(classUnderTest + " string must must have same hashCode as the flat string", + flatString.hashCode(), unicode.hashCode()); + } } diff --git a/java/src/test/java/com/google/protobuf/RopeByteStringTest.java b/java/src/test/java/com/google/protobuf/RopeByteStringTest.java index 0f2344d6..bd0d15e3 100644 --- a/java/src/test/java/com/google/protobuf/RopeByteStringTest.java +++ b/java/src/test/java/com/google/protobuf/RopeByteStringTest.java @@ -42,7 +42,7 @@ import java.util.Iterator; /** * This class tests {@link RopeByteString} by inheriting the tests from * {@link LiteralByteStringTest}. Only a couple of methods are overridden. - * + * * <p>A full test of the result of {@link RopeByteString#substring(int, int)} is found in the * separate class {@link RopeByteStringSubstringTest}. * @@ -119,11 +119,37 @@ public class RopeByteStringTest extends LiteralByteStringTest { } @Override + public void testCharsetToString() throws UnsupportedEncodingException { + String sourceString = "I love unicode \u1234\u5678 characters"; + ByteString sourceByteString = ByteString.copyFromUtf8(sourceString); + int copies = 250; + + // By building the RopeByteString by concatenating, this is actually a fairly strenuous test. + StringBuilder builder = new StringBuilder(copies * sourceString.length()); + ByteString unicode = ByteString.EMPTY; + for (int i = 0; i < copies; ++i) { + builder.append(sourceString); + unicode = RopeByteString.concatenate(unicode, sourceByteString); + } + String testString = builder.toString(); + + assertEquals(classUnderTest + " from string must have the expected type", + classUnderTest, getActualClassName(unicode)); + String roundTripString = unicode.toString(Internal.UTF_8); + assertEquals(classUnderTest + " unicode bytes must match", + testString, roundTripString); + ByteString flatString = ByteString.copyFromUtf8(testString); + assertEquals(classUnderTest + " string must equal the flat string", flatString, unicode); + assertEquals(classUnderTest + " string must must have same hashCode as the flat string", + flatString.hashCode(), unicode.hashCode()); + } + + @Override public void testToString_returnsCanonicalEmptyString() throws UnsupportedEncodingException { RopeByteString ropeByteString = RopeByteString.newInstanceForTest(ByteString.EMPTY, ByteString.EMPTY); assertSame(classUnderTest + " must be the same string references", - ByteString.EMPTY.toString(UTF_8), ropeByteString.toString(UTF_8)); + ByteString.EMPTY.toString(Internal.UTF_8), ropeByteString.toString(Internal.UTF_8)); } public void testToString_raisesException() throws UnsupportedEncodingException{ diff --git a/java/src/test/java/com/google/protobuf/TestUtil.java b/java/src/test/java/com/google/protobuf/TestUtil.java index 135a1174..19a96d0e 100644 --- a/java/src/test/java/com/google/protobuf/TestUtil.java +++ b/java/src/test/java/com/google/protobuf/TestUtil.java @@ -276,11 +276,7 @@ public final class TestUtil { /** Helper to convert a String to ByteString. */ static ByteString toBytes(String str) { - try { - return ByteString.copyFrom(str.getBytes("UTF-8")); - } catch(java.io.UnsupportedEncodingException e) { - throw new RuntimeException("UTF-8 not supported.", e); - } + return ByteString.copyFrom(str.getBytes(Internal.UTF_8)); } /** diff --git a/java/src/test/java/com/google/protobuf/TextFormatTest.java b/java/src/test/java/com/google/protobuf/TextFormatTest.java index eba06ca0..5d846646 100644 --- a/java/src/test/java/com/google/protobuf/TextFormatTest.java +++ b/java/src/test/java/com/google/protobuf/TextFormatTest.java @@ -243,8 +243,8 @@ public class TextFormatTest extends TestCase { * characters. The characters are converted directly to bytes, *not* * encoded using UTF-8. */ - private ByteString bytes(String str) throws Exception { - return ByteString.copyFrom(str.getBytes("ISO-8859-1")); + private ByteString bytes(String str) { + return ByteString.copyFrom(str.getBytes(Internal.ISO_8859_1)); } /** @@ -863,7 +863,7 @@ public class TextFormatTest extends TestCase { TextFormat.merge(TextFormat.printToUnicodeString(message), builder); assertEquals(message.getOptionalString(), builder.getOptionalString()); } - + public void testPrintToUnicodeStringWithNewlines() throws Exception { // No newlines at start and end assertEquals("optional_string: \"test newlines\\n\\nin\\nstring\"\n", diff --git a/java/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java b/java/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java index cec3da1e..e76b4a67 100644 --- a/java/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java +++ b/java/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java @@ -229,7 +229,7 @@ public class UnknownFieldSetLiteTest extends TestCase { public void testMalformedBytes() throws Exception { try { - Foo.parseFrom("this is a malformed protocol buffer".getBytes("UTF-8")); + Foo.parseFrom("this is a malformed protocol buffer".getBytes(Internal.UTF_8)); fail(); } catch (InvalidProtocolBufferException e) { // Expected. diff --git a/javanano/README.txt b/javanano/README.md index cfb3c3b4..7d696aa7 100644 --- a/javanano/README.txt +++ b/javanano/README.md @@ -1,10 +1,14 @@ Protocol Buffers - Google's data interchange format +=================================================== + +[](https://travis-ci.org/google/protobuf) + Copyright 2008 Google Inc. This directory contains the Java Protocol Buffers Nano runtime library. Installation - With Maven -========================= +------------------------- The Protocol Buffers build is managed using Maven. If you would rather build without Maven, see below. @@ -41,7 +45,7 @@ rather build without Maven, see below. The .jar will be placed in the "target" directory. Installation - Without Maven -============================ +---------------------------- If you would rather not install Maven to build the library, you may follow these instructions instead. Note that these instructions skip @@ -66,7 +70,7 @@ running unit tests. 4) Install the classes wherever you prefer. Nano version -============================ +------------ JavaNano is a special code generator and runtime library designed specially for resource-restricted systems, like Android. It is very resource-friendly in both @@ -130,7 +134,9 @@ penalty. This is not a problem if the field has no default or is an empty default. Nano Generator options +---------------------- +``` java_package -> <file-name>|<package-name> java_outer_classname -> <file-name>|<package-name> java_multiple_files -> true or false @@ -139,8 +145,10 @@ optional_field_style -> default or accessors enum_style -> c or java ignore_services -> true or false parcelable_messages -> true or false +``` + +**java_package=\<file-name\>|\<package-name\>** (no default) -java_package=<file-name>|<package-name> (no default) This allows overriding the 'java_package' option value for the given file from the command line. Use multiple java_package options to override the option for multiple @@ -149,7 +157,8 @@ java_package=<file-name>|<package-name> (no default) the same option defined in the file if present, or the proto package if present, or the default Java package. -java_outer_classname=<file-name>|<outer-classname> (no default) +**java_outer_classname=\<file-name\>|\<outer-classname\>** (no default) + This allows overriding the 'java_outer_classname' option for the given file from the command line. Use multiple java_outer_classname options to override the option for @@ -160,7 +169,8 @@ java_outer_classname=<file-name>|<outer-classname> (no default) outer class will nest all classes and integer constants generated from file-scope messages and enums. -java_multiple_files={true,false} (no default) +**java_multiple_files={true,false}** (no default) + This allows overriding the 'java_multiple_files' option in all source files and their imported files from the command line. The final value of this option for each @@ -178,10 +188,11 @@ java_multiple_files={true,false} (no default) incorrect references to the imported messages and enum constants. -java_nano_generate_has={true,false} (default: false) +**java_nano_generate_has={true,false}** (default: false) + DEPRECATED. Use optional_field_style=accessors. - If true, generates a public boolean variable has<fieldname> + If true, generates a public boolean variable has\<fieldname\> accompanying each optional or required field (not present for repeated fields, groups or messages). It is set to false initially and upon clear(). If parseFrom(...) reads the field from the wire, @@ -195,20 +206,21 @@ java_nano_generate_has={true,false} (default: false) many cases reading the default works and determining whether the field was received over the wire is irrelevant. -optional_field_style={default,accessors,reftypes} (default: default) +**optional_field_style={default,accessors,reftypes}** (default: default) + Defines the style of the generated code for fields. - * default * + * default In the default style, optional fields translate into public mutable Java fields, and the serialization process is as discussed in the "IMPORTANT" section above. - * accessors * + * accessors When set to 'accessors', each optional field is encapsulated behind - 4 accessors, namely get<fieldname>(), set<fieldname>(), has<fieldname>() - and clear<fieldname>() methods, with the standard semantics. The hazzer's + 4 accessors, namely get\<fieldname\>(), set\<fieldname\>(), has\<fieldname\>() + and clear\<fieldname\>() methods, with the standard semantics. The hazzer's return value determines whether a field is serialized, so this style is useful when you need to serialize a field with the default value, or check if a field has been explicitly set to its default value from the wire. @@ -224,7 +236,7 @@ optional_field_style={default,accessors,reftypes} (default: default) reducing the final code size. TODO(maxtroy): find ProGuard config that would work the best. - * reftypes * + * reftypes When set to 'reftypes', each proto field is generated as a public Java field. For primitive types, these fields use the Java reference types @@ -254,10 +266,11 @@ optional_field_style={default,accessors,reftypes} (default: default) required field (you have no reason to), you can only use java_nano_generate_has=true. -enum_style={c,java} (default: c) +**enum_style={c,java}** (default: c) + Defines where to put the int constants generated from enum members. - * c * + * c Use C-style, so the enum constants are available at the scope where the enum is defined. A file-scope enum's members are referenced like @@ -265,7 +278,7 @@ enum_style={c,java} (default: c) referenced as 'Message.ENUM_VALUE'. The enum name is unavailable. This complies with the Micro code generator's behavior. - * java * + * java Use Java-style, so the enum constants are available under the enum name and referenced like 'EnumName.ENUM_VALUE' (they are still int @@ -277,18 +290,21 @@ enum_style={c,java} (default: c) compiler inlines all referenced enum constants into the call sites, the interface remains unused and can be removed by ProGuard. -ignore_services={true,false} (default: false) +**ignore_services={true,false}** (default: false) + Skips services definitions. Nano doesn't support services. By default, if a service is defined it will generate a compilation error. If this flag is set to true, services will be silently ignored, instead. -parcelable_messages={true,false} (default: false) +**parcelable_messages={true,false}** (default: false) + Android-specific option to generate Parcelable messages. To use nano protobufs within the Android repo: +---------------------------------------------- - Set 'LOCAL_PROTOC_OPTIMIZE_TYPE := nano' in your local .mk file. When building a Java library or an app (package) target, the build @@ -313,17 +329,20 @@ To use nano protobufs within the Android repo: the two above. To use nano protobufs outside of Android repo: +---------------------------------------------- - Link with the generated jar file - <protobuf-root>java/target/protobuf-java-2.3.0-nano.jar. + \<protobuf-root\>java/target/protobuf-java-2.3.0-nano.jar. - Invoke with --javanano_out, e.g.: - +``` ./protoc '--javanano_out=\ java_package=src/proto/simple-data.proto|my_package,\ java_outer_classname=src/proto/simple-data.proto|OuterName\ :.' src/proto/simple-data.proto +``` Contributing to nano: +--------------------- Please add/edit tests in NanoTest.java. @@ -348,7 +367,7 @@ Please run the following steps to test: - "make -j12" and check for build errors Usage -===== +----- The complete documentation for Protocol Buffers is available via the web at: diff --git a/javanano/src/main/java/com/google/protobuf/nano/CodedInputByteBufferNano.java b/javanano/src/main/java/com/google/protobuf/nano/CodedInputByteBufferNano.java index b4f20fde..4b45c6d2 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/CodedInputByteBufferNano.java +++ b/javanano/src/main/java/com/google/protobuf/nano/CodedInputByteBufferNano.java @@ -190,12 +190,12 @@ public final class CodedInputByteBufferNano { if (size <= (bufferSize - bufferPos) && size > 0) { // Fast path: We already have the bytes in a contiguous buffer, so // just copy directly from it. - final String result = new String(buffer, bufferPos, size, "UTF-8"); + final String result = new String(buffer, bufferPos, size, InternalNano.UTF_8); bufferPos += size; return result; } else { // Slow path: Build a byte array first then copy it. - return new String(readRawBytes(size), "UTF-8"); + return new String(readRawBytes(size), InternalNano.UTF_8); } } diff --git a/javanano/src/main/java/com/google/protobuf/nano/CodedOutputByteBufferNano.java b/javanano/src/main/java/com/google/protobuf/nano/CodedOutputByteBufferNano.java index 2777f34c..35907183 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/CodedOutputByteBufferNano.java +++ b/javanano/src/main/java/com/google/protobuf/nano/CodedOutputByteBufferNano.java @@ -31,7 +31,6 @@ package com.google.protobuf.nano; import java.io.IOException; -import java.io.UnsupportedEncodingException; /** * Encodes and writes protocol message fields. @@ -291,7 +290,7 @@ public final class CodedOutputByteBufferNano { // Unfortunately there does not appear to be any way to tell Java to encode // UTF-8 directly into our buffer, so we have to let it create its own byte // array and then copy. - final byte[] bytes = value.getBytes("UTF-8"); + final byte[] bytes = value.getBytes(InternalNano.UTF_8); writeRawVarint32(bytes.length); writeRawBytes(bytes); } @@ -603,13 +602,9 @@ public final class CodedOutputByteBufferNano { * {@code string} field. */ public static int computeStringSizeNoTag(final String value) { - try { - final byte[] bytes = value.getBytes("UTF-8"); - return computeRawVarint32Size(bytes.length) + - bytes.length; - } catch (UnsupportedEncodingException e) { - throw new RuntimeException("UTF-8 not supported."); - } + final byte[] bytes = value.getBytes(InternalNano.UTF_8); + return computeRawVarint32Size(bytes.length) + + bytes.length; } /** diff --git a/javanano/src/main/java/com/google/protobuf/nano/InternalNano.java b/javanano/src/main/java/com/google/protobuf/nano/InternalNano.java index 4a08bb58..e7ba8d12 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/InternalNano.java +++ b/javanano/src/main/java/com/google/protobuf/nano/InternalNano.java @@ -33,7 +33,7 @@ package com.google.protobuf.nano; import com.google.protobuf.nano.MapFactories.MapFactory; import java.io.IOException; -import java.io.UnsupportedEncodingException; +import java.nio.charset.Charset; import java.util.Arrays; import java.util.Map; import java.util.Map.Entry; @@ -67,6 +67,8 @@ public final class InternalNano { public static final int TYPE_SINT32 = 17; public static final int TYPE_SINT64 = 18; + protected static final Charset UTF_8 = Charset.forName("UTF-8"); + protected static final Charset ISO_8859_1 = Charset.forName("ISO-8859-1"); private InternalNano() {} @@ -111,14 +113,7 @@ public final class InternalNano { * generated code calls this automatically. */ public static String stringDefaultValue(String bytes) { - try { - return new String(bytes.getBytes("ISO-8859-1"), "UTF-8"); - } catch (UnsupportedEncodingException e) { - // This should never happen since all JVMs are required to implement - // both of the above character sets. - throw new IllegalStateException( - "Java VM does not support a standard character set.", e); - } + return new String(bytes.getBytes(ISO_8859_1), InternalNano.UTF_8); } /** @@ -130,14 +125,7 @@ public final class InternalNano { * embed raw bytes as a string literal with ISO-8859-1 encoding. */ public static byte[] bytesDefaultValue(String bytes) { - try { - return bytes.getBytes("ISO-8859-1"); - } catch (UnsupportedEncodingException e) { - // This should never happen since all JVMs are required to implement - // ISO-8859-1. - throw new IllegalStateException( - "Java VM does not support a standard character set.", e); - } + return bytes.getBytes(ISO_8859_1); } /** @@ -145,11 +133,7 @@ public final class InternalNano { * UnsupportedEncodingException to a RuntimeException. */ public static byte[] copyFromUtf8(final String text) { - try { - return text.getBytes("UTF-8"); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException("UTF-8 not supported?"); - } + return text.getBytes(InternalNano.UTF_8); } /** diff --git a/javanano/src/test/java/com/google/protobuf/nano/NanoTest.java b/javanano/src/test/java/com/google/protobuf/nano/NanoTest.java index 4cc77226..7de84310 100644 --- a/javanano/src/test/java/com/google/protobuf/nano/NanoTest.java +++ b/javanano/src/test/java/com/google/protobuf/nano/NanoTest.java @@ -458,7 +458,7 @@ public class NanoTest extends TestCase { assertFalse(msg.optionalBytes.length > 0); msg.optionalBytes = InternalNano.copyFromUtf8("hello"); assertTrue(msg.optionalBytes.length > 0); - assertEquals("hello", new String(msg.optionalBytes, "UTF-8")); + assertEquals("hello", new String(msg.optionalBytes, InternalNano.UTF_8)); msg.clear(); assertFalse(msg.optionalBytes.length > 0); msg.clear() @@ -476,7 +476,7 @@ public class NanoTest extends TestCase { TestAllTypesNano newMsg = TestAllTypesNano.parseFrom(result); assertTrue(newMsg.optionalBytes.length > 0); - assertEquals("bye", new String(newMsg.optionalBytes, "UTF-8")); + assertEquals("bye", new String(newMsg.optionalBytes, InternalNano.UTF_8)); } public void testNanoOptionalGroup() throws Exception { @@ -1346,14 +1346,14 @@ public class NanoTest extends TestCase { InternalNano.copyFromUtf8("bye"), InternalNano.copyFromUtf8("boo") }; - assertEquals("bye", new String(msg.repeatedBytes[1], "UTF-8")); - assertEquals("boo", new String(msg.repeatedBytes[2], "UTF-8")); + assertEquals("bye", new String(msg.repeatedBytes[1], InternalNano.UTF_8)); + assertEquals("boo", new String(msg.repeatedBytes[2], InternalNano.UTF_8)); msg.clear(); assertEquals(0, msg.repeatedBytes.length); msg.clear() .repeatedBytes = new byte[][] { InternalNano.copyFromUtf8("boo") }; assertEquals(1, msg.repeatedBytes.length); - assertEquals("boo", new String(msg.repeatedBytes[0], "UTF-8")); + assertEquals("boo", new String(msg.repeatedBytes[0], InternalNano.UTF_8)); msg.clear(); assertEquals(0, msg.repeatedBytes.length); @@ -1385,8 +1385,8 @@ public class NanoTest extends TestCase { newMsg = TestAllTypesNano.parseFrom(result); assertEquals(2, newMsg.repeatedBytes.length); - assertEquals("hello", new String(newMsg.repeatedBytes[0], "UTF-8")); - assertEquals("world", new String(newMsg.repeatedBytes[1], "UTF-8")); + assertEquals("hello", new String(newMsg.repeatedBytes[0], InternalNano.UTF_8)); + assertEquals("world", new String(newMsg.repeatedBytes[1], InternalNano.UTF_8)); } public void testNanoRepeatedGroup() throws Exception { @@ -2277,9 +2277,9 @@ public class NanoTest extends TestCase { assertTrue(52.0e3 == msg.defaultDouble); assertEquals(true, msg.defaultBool); assertEquals("hello", msg.defaultString); - assertEquals("world", new String(msg.defaultBytes, "UTF-8")); + assertEquals("world", new String(msg.defaultBytes, InternalNano.UTF_8)); assertEquals("dünya", msg.defaultStringNonascii); - assertEquals("dünyab", new String(msg.defaultBytesNonascii, "UTF-8")); + assertEquals("dünyab", new String(msg.defaultBytesNonascii, InternalNano.UTF_8)); assertEquals(TestAllTypesNano.BAR, msg.defaultNestedEnum); assertEquals(NanoOuterClass.FOREIGN_NANO_BAR, msg.defaultForeignEnum); assertEquals(UnittestImportNano.IMPORT_NANO_BAR, msg.defaultImportEnum); @@ -2385,7 +2385,7 @@ public class NanoTest extends TestCase { assertEquals(TestAllTypesNanoHas.FOO, newMsg.optionalNestedEnum); assertEquals(41, newMsg.defaultInt32); assertEquals("hello", newMsg.defaultString); - assertEquals("world", new String(newMsg.defaultBytes, "UTF-8")); + assertEquals("world", new String(newMsg.defaultBytes, InternalNano.UTF_8)); assertEquals(TestAllTypesNanoHas.BAR, newMsg.defaultNestedEnum); assertEquals(Float.NaN, newMsg.defaultFloatNan); assertEquals(0, newMsg.id); @@ -2567,7 +2567,7 @@ public class NanoTest extends TestCase { assertEquals(TestNanoAccessors.FOO, newMsg.getOptionalNestedEnum()); assertEquals(41, newMsg.getDefaultInt32()); assertEquals("hello", newMsg.getDefaultString()); - assertEquals("world", new String(newMsg.getDefaultBytes(), "UTF-8")); + assertEquals("world", new String(newMsg.getDefaultBytes(), InternalNano.UTF_8)); assertEquals(TestNanoAccessors.BAR, newMsg.getDefaultNestedEnum()); assertEquals(Float.NaN, newMsg.getDefaultFloatNan()); assertEquals(0, newMsg.id); diff --git a/protoc-artifacts/README.md b/protoc-artifacts/README.md new file mode 100644 index 00000000..3a530197 --- /dev/null +++ b/protoc-artifacts/README.md @@ -0,0 +1,120 @@ +# Build scripts that publish pre-compiled protoc artifacts +``protoc`` is the compiler for ``.proto`` files. It generates language bindings +for the messages and/or RPC services from ``.proto`` files. + +Because ``protoc`` is a native executable, the scripts under this directory +build and publish a ``protoc`` executable (a.k.a. artifact) to Maven +repositories. The artifact can be used by build automation tools so that users +would not need to compile and install ``protoc`` for their systems. + +## Versioning +The version of the ``protoc`` artifact must be the same as the version of the +Protobuf project. + +## Artifact name +The name of a published ``protoc`` artifact is in the following format: +``protoc-<version>-<os>-<arch>.exe``, e.g., ``protoc-3.0.0-alpha-3-windows-x86_64.exe``. + +## System requirement +Install [Apache Maven](http://maven.apache.org/) if you don't have it. + +The scripts only work under Unix-like environments, e.g., Linux, MacOSX, and +Cygwin or MinGW for Windows. Please see ``README.md`` of the Protobuf project +for how to set up the build environment. + +## To install artifacts locally +The following command will install the ``protoc`` artifact to your local Maven repository. +``` +$ mvn install +``` + +## Cross-compilation +The Maven script will try to detect the OS and the architecture from Java +system properties. It's possible to build a protoc binary for an architecture +that is different from what Java has detected, as long as you have the proper +compilers installed. + +You can override the Maven properties ``os.detected.name`` and +``os.detected.arch`` to force the script to generate binaries for a specific OS +and/or architecture. Valid values are defined as the return values of +``normalizeOs()`` and ``normalizeArch()`` of ``Detector`` from +[os-maven-plugin](https://github.com/trustin/os-maven-plugin/blob/master/src/main/java/kr/motd/maven/os/Detector.java). +Frequently used values are: +- ``os.detected.name``: ``linux``, ``osx``, ``windows``. +- ``os.detected.arch``: ``x86_32``, ``x86_64`` + +For example, MingGW32 only ships with 32-bit compilers, but you can still build +32-bit protoc under 64-bit Windows, with the following command: +``` +$ mvn install -Dos.detected.arch=x86_32 +``` + +## To push artifacts to Maven Central +Before you can upload artifacts to Maven Central repository, make sure you have +read [this page](http://central.sonatype.org/pages/apache-maven.html) on how to +configure GPG and Sonatype account. + +You need to perform the deployment for every platform that you want to +suppport. DO NOT close the staging repository until you have done the +deployment for all platforms. Currently the following platforms are supported: +- Linux (x86_32 and x86_64) +- Windows (x86_32 and x86_64) with + - Cygwin with MinGW compilers (both x86_32 and x86_64) + - MSYS with MinGW32 (x86_32 only) +- MacOSX (x86_32 and x86_64) + +Remove any ``SNAPSHOT`` or ``pre`` suffix from the version string before +deploying. + +Use the following command to deploy artifacts for the host platform to a +staging repository. +``` +$ mvn clean deploy -P release +``` +It creates a new staging repository. Go to +https://oss.sonatype.org/#stagingRepositories and find the repository, usually +in the name like ``comgoogle-123``. + +You will want to run this command on a different platform. Remember, in +subsequent deployments you will need to provide the repository name that you +have found in the first deployment so that all artifacts go to the same +repository: +``` +$ mvn clean deploy -P release -Dstaging.repository=comgoogle-123 +``` + +A 32-bit artifact can be deployed from a 64-bit host with +``-Dos.detected.arch=x86_32`` + +When you have done deployment for all platforms, go to +https://oss.sonatype.org/#stagingRepositories, verify that the staging +repository has all the binaries, close and release this repository. + +### Tips for deploying on Windows +Under Windows the following error may occur: ``gpg: cannot open tty `no tty': +No such file or directory``. This can be fixed by configuring gpg through an +active profile in ``.m2\settings.xml`` where also the Sonatype password is +stored: +```xml +<settings> + <servers> + <server> + <id>ossrh</id> + <username>[username]</username> + <password>[password]</password> + </server> + </servers> + <profiles> + <profile> + <id>gpg</id> + <properties> + <gpg.executable>gpg</gpg.executable> + <gpg.passphrase>[password]</gpg.passphrase> + </properties> + </profile> + </profiles> + <activeProfiles> + <activeProfile>gpg</activeProfile> + </activeProfiles> +</settings> +``` diff --git a/protoc-artifacts/build-protoc.sh b/protoc-artifacts/build-protoc.sh new file mode 100755 index 00000000..96ca97c2 --- /dev/null +++ b/protoc-artifacts/build-protoc.sh @@ -0,0 +1,222 @@ +#!/bin/bash + +# Builds protoc executable into target/protoc.exe +# To be run from Maven. +# Usage: build-protoc.sh <OS> <ARCH> +# <OS> and <ARCH> are ${os.detected.name} and ${os.detected.arch} from os-maven-plugin +OS=$1 +ARCH=$2 + +if [[ $# < 2 ]]; then + echo "No arguments provided. This script is intended to be run from Maven." + exit 1 +fi + +# Under Cygwin, bash doesn't have these in PATH when called from Maven which +# runs in Windows version of Java. +export PATH="/bin:/usr/bin:$PATH" + +############################################################################ +# Helper functions +############################################################################ +E_PARAM_ERR=98 +E_ASSERT_FAILED=99 + +# Usage: +fail() +{ + echo "ERROR: $1" + echo + exit $E_ASSERT_FAILED +} + +# Usage: assertEq VAL1 VAL2 $LINENO +assertEq () +{ + lineno=$3 + if [ -z "$lineno" ]; then + echo "lineno not given" + exit $E_PARAM_ERR + fi + + if [[ "$1" != "$2" ]]; then + echo "Assertion failed: \"$1\" == \"$2\"" + echo "File \"$0\", line $lineno" # Give name of file and line number. + exit $E_ASSERT_FAILED + fi +} + +# Checks the artifact is for the expected architecture +# Usage: checkArch <path-to-protoc> +checkArch () +{ + echo + echo "Checking file format ..." + if [[ "$OS" == windows || "$OS" == linux ]]; then + format="$(objdump -f "$1" | grep -o "file format .*$" | grep -o "[^ ]*$")" + echo Format=$format + if [[ "$OS" == linux ]]; then + if [[ "$ARCH" == x86_32 ]]; then + assertEq $format "elf32-i386" $LINENO + elif [[ "$ARCH" == x86_64 ]]; then + assertEq $format "elf64-x86-64" $LINENO + else + fail "Unsupported arch: $ARCH" + fi + else + # $OS == windows + if [[ "$ARCH" == x86_32 ]]; then + assertEq $format "pei-i386" $LINENO + elif [[ "$ARCH" == x86_64 ]]; then + assertEq $format "pei-x86-64" $LINENO + else + fail "Unsupported arch: $ARCH" + fi + fi + elif [[ "$OS" == osx ]]; then + format="$(file -b "$1" | grep -o "[^ ]*$")" + echo Format=$format + if [[ "$ARCH" == x86_32 ]]; then + assertEq $format "i386" $LINENO + elif [[ "$ARCH" == x86_64 ]]; then + assertEq $format "x86_64" $LINENO + else + fail "Unsupported arch: $ARCH" + fi + else + fail "Unsupported system: $OS" + fi + echo +} + +# Checks the dependencies of the artifact. Artifacts should only depend on +# system libraries. +# Usage: checkDependencies <path-to-protoc> +checkDependencies () +{ + if [[ "$OS" == windows ]]; then + dump_cmd='objdump -x '"$1"' | fgrep "DLL Name"' + white_list="KERNEL32\.dll\|msvcrt\.dll" + elif [[ "$OS" == linux ]]; then + dump_cmd='ldd '"$1" + if [[ "$ARCH" == x86_32 ]]; then + white_list="linux-gate\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|ld-linux\.so\.2" + elif [[ "$ARCH" == x86_64 ]]; then + white_list="linux-vdso\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|ld-linux-x86-64\.so\.2" + fi + elif [[ "$OS" == osx ]]; then + dump_cmd='otool -L '"$1"' | fgrep dylib' + white_list="libz\.1\.dylib\|libstdc++\.6\.dylib\|libSystem\.B\.dylib" + fi + if [[ -z "$white_list" || -z "$dump_cmd" ]]; then + fail "Unsupported platform $OS-$ARCH." + fi + echo "Checking for expected dependencies ..." + eval $dump_cmd | grep -i "$white_list" || fail "doesn't show any expected dependencies" + echo "Checking for unexpected dependencies ..." + eval $dump_cmd | grep -i -v "$white_list" + ret=$? + if [[ $ret == 0 ]]; then + fail "found unexpected dependencies (listed above)." + elif [[ $ret != 1 ]]; then + fail "Error when checking dependencies." + fi # grep returns 1 when "not found", which is what we expect + echo "Dependencies look good." + echo +} +############################################################################ + +echo "Building protoc, OS=$OS ARCH=$ARCH" + +# Nested double quotes are unintuitive, but it works. +cd "$(dirname "$0")" + +WORKING_DIR=$(pwd) +CONFIGURE_ARGS="--disable-shared" + +MAKE_TARGET="protoc" +if [[ "$OS" == windows ]]; then + MAKE_TARGET="${MAKE_TARGET}.exe" +fi + +# Override the default value set in configure.ac that has '-g' which produces +# huge binary. +CXXFLAGS="-DNDEBUG" +LDFLAGS="" + +if [[ "$(uname)" == CYGWIN* ]]; then + assertEq "$OS" windows $LINENO + # Use mingw32 compilers because executables produced by Cygwin compiler + # always have dependency on Cygwin DLL. + if [[ "$ARCH" == x86_64 ]]; then + CONFIGURE_ARGS="$CONFIGURE_ARGS --host=x86_64-w64-mingw32" + elif [[ "$ARCH" == x86_32 ]]; then + CONFIGURE_ARGS="$CONFIGURE_ARGS --host=i686-pc-mingw32" + else + fail "Unsupported arch by CYGWIN: $ARCH" + fi +elif [[ "$(uname)" == MINGW32* ]]; then + assertEq "$OS" windows $LINENO + assertEq "$ARCH" x86_32 $LINENO +elif [[ "$(uname)" == Linux* ]]; then + if [[ "$OS" == linux ]]; then + if [[ "$ARCH" == x86_64 ]]; then + CXXFLAGS="$CXXFLAGS -m64" + elif [[ "$ARCH" == x86_32 ]]; then + CXXFLAGS="$CXXFLAGS -m32" + else + fail "Unsupported arch: $ARCH" + fi + elif [[ "$OS" == windows ]]; then + # Cross-compilation for Windows + # TODO(zhangkun83) MinGW 64 always adds dependency on libwinpthread-1.dll, + # which is undesirable for repository deployment. + CONFIGURE_ARGS="$CONFIGURE_ARGS" + if [[ "$ARCH" == x86_64 ]]; then + CONFIGURE_ARGS="$CONFIGURE_ARGS --host=x86_64-w64-mingw32" + elif [[ "$ARCH" == x86_32 ]]; then + CONFIGURE_ARGS="$CONFIGURE_ARGS --host=i686-w64-mingw32" + else + fail "Unsupported arch: $ARCH" + fi + else + fail "Cannot build $OS on $(uname)" + fi +elif [[ "$(uname)" == Darwin* ]]; then + assertEq "$OS" osx $LINENO + # Make the binary compatible with OSX 10.7 and later + CXXFLAGS="$CXXFLAGS -mmacosx-version-min=10.7" + if [[ "$ARCH" == x86_64 ]]; then + CXXFLAGS="$CXXFLAGS -m64" + elif [[ "$ARCH" == x86_32 ]]; then + CXXFLAGS="$CXXFLAGS -m32" + else + fail "Unsupported arch: $ARCH" + fi +else + fail "Unsupported system: $(uname)" +fi + +# Statically link libgcc and libstdc++. +# -s to produce stripped binary. +# And they don't work under Mac. +if [[ "$OS" != osx ]]; then + LDFLAGS="$LDFLAGS -static-libgcc -static-libstdc++ -s" +fi + +export CXXFLAGS LDFLAGS + +TARGET_FILE=target/protoc.exe + +cd "$WORKING_DIR"/.. && ./configure $CONFIGURE_ARGS && + cd src && make clean && make $MAKE_TARGET && + cd "$WORKING_DIR" && mkdir -p target && + (cp ../src/protoc $TARGET_FILE || cp ../src/protoc.exe $TARGET_FILE) || + exit 1 + +if [[ "$OS" == osx ]]; then + # Since Mac linker doesn't accept "-s", we need to run strip + strip $TARGET_FILE || exit 1 +fi + +checkArch $TARGET_FILE && checkDependencies $TARGET_FILE diff --git a/protoc-artifacts/pom.xml b/protoc-artifacts/pom.xml new file mode 100644 index 00000000..fad81b57 --- /dev/null +++ b/protoc-artifacts/pom.xml @@ -0,0 +1,134 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.google</groupId> + <artifactId>google</artifactId> + <version>1</version> + </parent> + <groupId>com.google.protobuf</groupId> + <artifactId>protoc</artifactId> + <version>3.0.0-alpha-3-pre</version> + <packaging>pom</packaging> + <name>Protobuf Compiler</name> + <description> + Protobuf Compiler (protoc) is a compiler for .proto files. It generates + language-specific code for Protobuf messages and RPC interfaces. + </description> + <inceptionYear>2008</inceptionYear> + <url>https://developers.google.com/protocol-buffers/</url> + <licenses> + <license> + <name>New BSD license</name> + <url>http://www.opensource.org/licenses/bsd-license.php</url> + <distribution>repo</distribution> + </license> + </licenses> + <scm> + <url>https://github.com/google/protobuf</url> + <connection> + scm:git:https://github.com/google/protobuf.git + </connection> + </scm> + <build> + <extensions> + <extension> + <groupId>kr.motd.maven</groupId> + <artifactId>os-maven-plugin</artifactId> + <version>1.2.3.Final</version> + </extension> + </extensions> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>1.1.1</version> + <executions> + <execution> + <phase>compile</phase> + <goals> + <goal>exec</goal> + </goals> + </execution> + </executions> + <configuration> + <executable>bash</executable> + <arguments> + <argument>build-protoc.sh</argument> + <argument>${os.detected.name}</argument> + <argument>${os.detected.arch}</argument> + </arguments> + </configuration> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>1.8</version> + <executions> + <execution> + <id>attach-artifacts</id> + <phase>package</phase> + <goals> + <goal>attach-artifact</goal> + </goals> + <configuration> + <artifacts> + <artifact> + <file>${basedir}/target/protoc.exe</file> + <classifier>${os.detected.name}-${os.detected.arch}</classifier> + <type>exe</type> + </artifact> + </artifacts> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + <profiles> + <profile> + <id>release</id> + <properties> + <!-- Specify the staging repository to deploy to. This can be left + empty for the first deployment, and Sonatype will create one. For + subsequent deployments it should be set to what Sonatype has + created, so that all deployments will go to the same repository. + --> + <staging.repository></staging.repository> + </properties> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-gpg-plugin</artifactId> + <version>1.5</version> + <executions> + <execution> + <id>sign-artifacts</id> + <phase>verify</phase> + <goals> + <goal>sign</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.sonatype.plugins</groupId> + <artifactId>nexus-staging-maven-plugin</artifactId> + <version>1.6.3</version> + <extensions>true</extensions> + <configuration> + <serverId>sonatype-nexus-staging</serverId> + <nexusUrl>https://oss.sonatype.org/</nexusUrl> + <skipStagingRepositoryClose>true</skipStagingRepositoryClose> + <autoReleaseAfterClose>false</autoReleaseAfterClose> + <stagingRepositoryId>${staging.repository}</stagingRepositoryId> + </configuration> + </plugin> + </plugins> + </build> + </profile> + </profiles> +</project> diff --git a/python/README.txt b/python/README.md index 04cb1767..1b5b9dff 100644 --- a/python/README.txt +++ b/python/README.md @@ -1,4 +1,8 @@ Protocol Buffers - Google's data interchange format +=================================================== + +[](https://travis-ci.org/google/protobuf) + Copyright 2008 Google Inc. This directory contains the Python Protocol Buffers runtime library. @@ -46,11 +50,39 @@ Installation 4) Build and run the tests: $ python setup.py build - $ python setup.py google_test + $ python setup.py test + + To build, test, and use the C++ implementation, you must first compile + libprotobuf.so: + + $ (cd .. && make) + + On OS X: + + If you are running a homebrew-provided python, you must make sure another + version of protobuf is not already installed, as homebrew's python will + search /usr/local/lib for libprotobuf.so before it searches ../src/.libs + You can either unlink homebrew's protobuf or install the libprotobuf you + built earlier: + + $ brew unlink protobuf + or + $ (cd .. && make install) - If you want to build/test c++ implementation, run: + On other *nix: + + You must make libprotobuf.so dynamically available. You can either + install libprotobuf you built earlier, or set LD_LIBRARY_PATH: + + $ export LD_LIBRARY_PATH=../src/.libs + or + $ (cd .. && make install) + + To build the C++ implementation run: $ python setup.py build --cpp_implementation - $ python setup.py google_test --cpp_implementation + + Then run the tests like so: + $ python setup.py test --cpp_implementation If some tests fail, this library may not work correctly on your system. Continue at your own risk. @@ -65,14 +97,17 @@ Installation 5) Install: - $ python setup.py install - or: - $ python setup.py install --cpp_implementation + $ python setup.py install + + or: + + $ (cd .. && make install) + $ python setup.py install --cpp_implementation This step may require superuser privileges. - NOTE: To use C++ implementation, you need to install C++ protobuf runtime - library of the same version and export the environment variable before this - step. See the "C++ Implementation" section below for more details. + NOTE: To use C++ implementation, you need to export an environment + variable before running your program. See the "C++ Implementation" + section below for more details. Usage ===== @@ -93,14 +128,8 @@ To use the C++ implementation, you need to: parent directory. 2) Export an environment variable: - $ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp - $ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION=2 + $ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp -You need to export this variable before running setup.py script to build and -install the extension. You must also set the variable at runtime, otherwise +You must set this variable at runtime, before running your program, otherwise the pure-Python implementation will be used. In a future release, we will change the default so that C++ implementation is used whenever it is available. -It is strongly recommended to run `python setup.py test` after setting the -variable to "cpp", so the tests will be against C++ implemented Python -messages. - diff --git a/python/google/protobuf/internal/_parameterized.py b/python/google/protobuf/internal/_parameterized.py index 6ed23308..c44e77e4 100755 --- a/python/google/protobuf/internal/_parameterized.py +++ b/python/google/protobuf/internal/_parameterized.py @@ -152,8 +152,6 @@ import types import unittest import uuid -from google.apputils import basetest - ADDR_RE = re.compile(r'\<([a-zA-Z0-9_\-\.]+) object at 0x[a-fA-F0-9]+\>') _SEPARATOR = uuid.uuid1().hex _FIRST_ARG = object() @@ -380,7 +378,7 @@ def _UpdateClassDictForParamTestCase(dct, id_suffix, name, iterator): id_suffix[new_name] = getattr(func, '__x_extra_id__', '') -class ParameterizedTestCase(basetest.TestCase): +class ParameterizedTestCase(unittest.TestCase): """Base class for test cases using the Parameters decorator.""" __metaclass__ = TestGeneratorMetaclass diff --git a/python/google/protobuf/internal/api_implementation.py b/python/google/protobuf/internal/api_implementation.py index 8ba4357c..ffcf7511 100755 --- a/python/google/protobuf/internal/api_implementation.py +++ b/python/google/protobuf/internal/api_implementation.py @@ -80,8 +80,8 @@ if _implementation_type != 'python': # This environment variable can be used to switch between the two # 'cpp' implementations, overriding the compile-time constants in the -# _api_implementation module. Right now only 1 and 2 are valid values. Any other -# value will be ignored. +# _api_implementation module. Right now only '2' is supported. Any other +# value will cause an error to be raised. _implementation_version_str = os.getenv( 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION', '2') diff --git a/python/google/protobuf/internal/cpp_message.py b/python/google/protobuf/internal/cpp_message.py deleted file mode 100755 index 0313cb0b..00000000 --- a/python/google/protobuf/internal/cpp_message.py +++ /dev/null @@ -1,663 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Contains helper functions used to create protocol message classes from -Descriptor objects at runtime backed by the protocol buffer C++ API. -""" - -__author__ = 'petar@google.com (Petar Petrov)' - -import copy_reg -import operator -from google.protobuf.internal import _net_proto2___python -from google.protobuf.internal import enum_type_wrapper -from google.protobuf import message - - -_LABEL_REPEATED = _net_proto2___python.LABEL_REPEATED -_LABEL_OPTIONAL = _net_proto2___python.LABEL_OPTIONAL -_CPPTYPE_MESSAGE = _net_proto2___python.CPPTYPE_MESSAGE -_TYPE_MESSAGE = _net_proto2___python.TYPE_MESSAGE - - -def GetDescriptorPool(): - """Creates a new DescriptorPool C++ object.""" - return _net_proto2___python.NewCDescriptorPool() - - -_pool = GetDescriptorPool() - - -def GetFieldDescriptor(full_field_name): - """Searches for a field descriptor given a full field name.""" - return _pool.FindFieldByName(full_field_name) - - -def BuildFile(content): - """Registers a new proto file in the underlying C++ descriptor pool.""" - _net_proto2___python.BuildFile(content) - - -def GetExtensionDescriptor(full_extension_name): - """Searches for extension descriptor given a full field name.""" - return _pool.FindExtensionByName(full_extension_name) - - -def NewCMessage(full_message_name): - """Creates a new C++ protocol message by its name.""" - return _net_proto2___python.NewCMessage(full_message_name) - - -def ScalarProperty(cdescriptor): - """Returns a scalar property for the given descriptor.""" - - def Getter(self): - return self._cmsg.GetScalar(cdescriptor) - - def Setter(self, value): - self._cmsg.SetScalar(cdescriptor, value) - - return property(Getter, Setter) - - -def CompositeProperty(cdescriptor, message_type): - """Returns a Python property the given composite field.""" - - def Getter(self): - sub_message = self._composite_fields.get(cdescriptor.name, None) - if sub_message is None: - cmessage = self._cmsg.NewSubMessage(cdescriptor) - sub_message = message_type._concrete_class(__cmessage=cmessage) - self._composite_fields[cdescriptor.name] = sub_message - return sub_message - - return property(Getter) - - -class RepeatedScalarContainer(object): - """Container for repeated scalar fields.""" - - __slots__ = ['_message', '_cfield_descriptor', '_cmsg'] - - def __init__(self, msg, cfield_descriptor): - self._message = msg - self._cmsg = msg._cmsg - self._cfield_descriptor = cfield_descriptor - - def append(self, value): - self._cmsg.AddRepeatedScalar( - self._cfield_descriptor, value) - - def extend(self, sequence): - for element in sequence: - self.append(element) - - def insert(self, key, value): - values = self[slice(None, None, None)] - values.insert(key, value) - self._cmsg.AssignRepeatedScalar(self._cfield_descriptor, values) - - def remove(self, value): - values = self[slice(None, None, None)] - values.remove(value) - self._cmsg.AssignRepeatedScalar(self._cfield_descriptor, values) - - def __setitem__(self, key, value): - values = self[slice(None, None, None)] - values[key] = value - self._cmsg.AssignRepeatedScalar(self._cfield_descriptor, values) - - def __getitem__(self, key): - return self._cmsg.GetRepeatedScalar(self._cfield_descriptor, key) - - def __delitem__(self, key): - self._cmsg.DeleteRepeatedField(self._cfield_descriptor, key) - - def __len__(self): - return len(self[slice(None, None, None)]) - - def __eq__(self, other): - if self is other: - return True - if not operator.isSequenceType(other): - raise TypeError( - 'Can only compare repeated scalar fields against sequences.') - # We are presumably comparing against some other sequence type. - return other == self[slice(None, None, None)] - - def __ne__(self, other): - return not self == other - - def __hash__(self): - raise TypeError('unhashable object') - - def sort(self, *args, **kwargs): - # Maintain compatibility with the previous interface. - if 'sort_function' in kwargs: - kwargs['cmp'] = kwargs.pop('sort_function') - self._cmsg.AssignRepeatedScalar(self._cfield_descriptor, - sorted(self, *args, **kwargs)) - - -def RepeatedScalarProperty(cdescriptor): - """Returns a Python property the given repeated scalar field.""" - - def Getter(self): - container = self._composite_fields.get(cdescriptor.name, None) - if container is None: - container = RepeatedScalarContainer(self, cdescriptor) - self._composite_fields[cdescriptor.name] = container - return container - - def Setter(self, new_value): - raise AttributeError('Assignment not allowed to repeated field ' - '"%s" in protocol message object.' % cdescriptor.name) - - doc = 'Magic attribute generated for "%s" proto field.' % cdescriptor.name - return property(Getter, Setter, doc=doc) - - -class RepeatedCompositeContainer(object): - """Container for repeated composite fields.""" - - __slots__ = ['_message', '_subclass', '_cfield_descriptor', '_cmsg'] - - def __init__(self, msg, cfield_descriptor, subclass): - self._message = msg - self._cmsg = msg._cmsg - self._subclass = subclass - self._cfield_descriptor = cfield_descriptor - - def add(self, **kwargs): - cmessage = self._cmsg.AddMessage(self._cfield_descriptor) - return self._subclass(__cmessage=cmessage, __owner=self._message, **kwargs) - - def extend(self, elem_seq): - """Extends by appending the given sequence of elements of the same type - as this one, copying each individual message. - """ - for message in elem_seq: - self.add().MergeFrom(message) - - def remove(self, value): - # TODO(protocol-devel): This is inefficient as it needs to generate a - # message pointer for each message only to do index(). Move this to a C++ - # extension function. - self.__delitem__(self[slice(None, None, None)].index(value)) - - def MergeFrom(self, other): - for message in other[:]: - self.add().MergeFrom(message) - - def __getitem__(self, key): - cmessages = self._cmsg.GetRepeatedMessage( - self._cfield_descriptor, key) - subclass = self._subclass - if not isinstance(cmessages, list): - return subclass(__cmessage=cmessages, __owner=self._message) - - return [subclass(__cmessage=m, __owner=self._message) for m in cmessages] - - def __delitem__(self, key): - self._cmsg.DeleteRepeatedField( - self._cfield_descriptor, key) - - def __len__(self): - return self._cmsg.FieldLength(self._cfield_descriptor) - - def __eq__(self, other): - """Compares the current instance with another one.""" - if self is other: - return True - if not isinstance(other, self.__class__): - raise TypeError('Can only compare repeated composite fields against ' - 'other repeated composite fields.') - messages = self[slice(None, None, None)] - other_messages = other[slice(None, None, None)] - return messages == other_messages - - def __hash__(self): - raise TypeError('unhashable object') - - def sort(self, cmp=None, key=None, reverse=False, **kwargs): - # Maintain compatibility with the old interface. - if cmp is None and 'sort_function' in kwargs: - cmp = kwargs.pop('sort_function') - - # The cmp function, if provided, is passed the results of the key function, - # so we only need to wrap one of them. - if key is None: - index_key = self.__getitem__ - else: - index_key = lambda i: key(self[i]) - - # Sort the list of current indexes by the underlying object. - indexes = range(len(self)) - indexes.sort(cmp=cmp, key=index_key, reverse=reverse) - - # Apply the transposition. - for dest, src in enumerate(indexes): - if dest == src: - continue - self._cmsg.SwapRepeatedFieldElements(self._cfield_descriptor, dest, src) - # Don't swap the same value twice. - indexes[src] = src - - -def RepeatedCompositeProperty(cdescriptor, message_type): - """Returns a Python property for the given repeated composite field.""" - - def Getter(self): - container = self._composite_fields.get(cdescriptor.name, None) - if container is None: - container = RepeatedCompositeContainer( - self, cdescriptor, message_type._concrete_class) - self._composite_fields[cdescriptor.name] = container - return container - - def Setter(self, new_value): - raise AttributeError('Assignment not allowed to repeated field ' - '"%s" in protocol message object.' % cdescriptor.name) - - doc = 'Magic attribute generated for "%s" proto field.' % cdescriptor.name - return property(Getter, Setter, doc=doc) - - -class ExtensionDict(object): - """Extension dictionary added to each protocol message.""" - - def __init__(self, msg): - self._message = msg - self._cmsg = msg._cmsg - self._values = {} - - def __setitem__(self, extension, value): - from google.protobuf import descriptor - if not isinstance(extension, descriptor.FieldDescriptor): - raise KeyError('Bad extension %r.' % (extension,)) - cdescriptor = extension._cdescriptor - if (cdescriptor.label != _LABEL_OPTIONAL or - cdescriptor.cpp_type == _CPPTYPE_MESSAGE): - raise TypeError('Extension %r is repeated and/or a composite type.' % ( - extension.full_name,)) - self._cmsg.SetScalar(cdescriptor, value) - self._values[extension] = value - - def __getitem__(self, extension): - from google.protobuf import descriptor - if not isinstance(extension, descriptor.FieldDescriptor): - raise KeyError('Bad extension %r.' % (extension,)) - - cdescriptor = extension._cdescriptor - if (cdescriptor.label != _LABEL_REPEATED and - cdescriptor.cpp_type != _CPPTYPE_MESSAGE): - return self._cmsg.GetScalar(cdescriptor) - - ext = self._values.get(extension, None) - if ext is not None: - return ext - - ext = self._CreateNewHandle(extension) - self._values[extension] = ext - return ext - - def ClearExtension(self, extension): - from google.protobuf import descriptor - if not isinstance(extension, descriptor.FieldDescriptor): - raise KeyError('Bad extension %r.' % (extension,)) - self._cmsg.ClearFieldByDescriptor(extension._cdescriptor) - if extension in self._values: - del self._values[extension] - - def HasExtension(self, extension): - from google.protobuf import descriptor - if not isinstance(extension, descriptor.FieldDescriptor): - raise KeyError('Bad extension %r.' % (extension,)) - return self._cmsg.HasFieldByDescriptor(extension._cdescriptor) - - def _FindExtensionByName(self, name): - """Tries to find a known extension with the specified name. - - Args: - name: Extension full name. - - Returns: - Extension field descriptor. - """ - return self._message._extensions_by_name.get(name, None) - - def _CreateNewHandle(self, extension): - cdescriptor = extension._cdescriptor - if (cdescriptor.label != _LABEL_REPEATED and - cdescriptor.cpp_type == _CPPTYPE_MESSAGE): - cmessage = self._cmsg.NewSubMessage(cdescriptor) - return extension.message_type._concrete_class(__cmessage=cmessage) - - if cdescriptor.label == _LABEL_REPEATED: - if cdescriptor.cpp_type == _CPPTYPE_MESSAGE: - return RepeatedCompositeContainer( - self._message, cdescriptor, extension.message_type._concrete_class) - else: - return RepeatedScalarContainer(self._message, cdescriptor) - # This shouldn't happen! - assert False - return None - - -def NewMessage(bases, message_descriptor, dictionary): - """Creates a new protocol message *class*.""" - _AddClassAttributesForNestedExtensions(message_descriptor, dictionary) - _AddEnumValues(message_descriptor, dictionary) - _AddDescriptors(message_descriptor, dictionary) - return bases - - -def InitMessage(message_descriptor, cls): - """Constructs a new message instance (called before instance's __init__).""" - cls._extensions_by_name = {} - _AddInitMethod(message_descriptor, cls) - _AddMessageMethods(message_descriptor, cls) - _AddPropertiesForExtensions(message_descriptor, cls) - copy_reg.pickle(cls, lambda obj: (cls, (), obj.__getstate__())) - - -def _AddDescriptors(message_descriptor, dictionary): - """Sets up a new protocol message class dictionary. - - Args: - message_descriptor: A Descriptor instance describing this message type. - dictionary: Class dictionary to which we'll add a '__slots__' entry. - """ - dictionary['__descriptors'] = {} - for field in message_descriptor.fields: - dictionary['__descriptors'][field.name] = GetFieldDescriptor( - field.full_name) - - dictionary['__slots__'] = list(dictionary['__descriptors'].iterkeys()) + [ - '_cmsg', '_owner', '_composite_fields', 'Extensions', '_HACK_REFCOUNTS'] - - -def _AddEnumValues(message_descriptor, dictionary): - """Sets class-level attributes for all enum fields defined in this message. - - Args: - message_descriptor: Descriptor object for this message type. - dictionary: Class dictionary that should be populated. - """ - for enum_type in message_descriptor.enum_types: - dictionary[enum_type.name] = enum_type_wrapper.EnumTypeWrapper(enum_type) - for enum_value in enum_type.values: - dictionary[enum_value.name] = enum_value.number - - -def _AddClassAttributesForNestedExtensions(message_descriptor, dictionary): - """Adds class attributes for the nested extensions.""" - extension_dict = message_descriptor.extensions_by_name - for extension_name, extension_field in extension_dict.iteritems(): - assert extension_name not in dictionary - dictionary[extension_name] = extension_field - - -def _AddInitMethod(message_descriptor, cls): - """Adds an __init__ method to cls.""" - - # Create and attach message field properties to the message class. - # This can be done just once per message class, since property setters and - # getters are passed the message instance. - # This makes message instantiation extremely fast, and at the same time it - # doesn't require the creation of property objects for each message instance, - # which saves a lot of memory. - for field in message_descriptor.fields: - field_cdescriptor = cls.__descriptors[field.name] - if field.label == _LABEL_REPEATED: - if field.cpp_type == _CPPTYPE_MESSAGE: - value = RepeatedCompositeProperty(field_cdescriptor, field.message_type) - else: - value = RepeatedScalarProperty(field_cdescriptor) - elif field.cpp_type == _CPPTYPE_MESSAGE: - value = CompositeProperty(field_cdescriptor, field.message_type) - else: - value = ScalarProperty(field_cdescriptor) - setattr(cls, field.name, value) - - # Attach a constant with the field number. - constant_name = field.name.upper() + '_FIELD_NUMBER' - setattr(cls, constant_name, field.number) - - def Init(self, **kwargs): - """Message constructor.""" - cmessage = kwargs.pop('__cmessage', None) - if cmessage: - self._cmsg = cmessage - else: - self._cmsg = NewCMessage(message_descriptor.full_name) - - # Keep a reference to the owner, as the owner keeps a reference to the - # underlying protocol buffer message. - owner = kwargs.pop('__owner', None) - if owner: - self._owner = owner - - if message_descriptor.is_extendable: - self.Extensions = ExtensionDict(self) - else: - # Reference counting in the C++ code is broken and depends on - # the Extensions reference to keep this object alive during unit - # tests (see b/4856052). Remove this once b/4945904 is fixed. - self._HACK_REFCOUNTS = self - self._composite_fields = {} - - for field_name, field_value in kwargs.iteritems(): - field_cdescriptor = self.__descriptors.get(field_name, None) - if not field_cdescriptor: - raise ValueError('Protocol message has no "%s" field.' % field_name) - if field_cdescriptor.label == _LABEL_REPEATED: - if field_cdescriptor.cpp_type == _CPPTYPE_MESSAGE: - field_name = getattr(self, field_name) - for val in field_value: - field_name.add().MergeFrom(val) - else: - getattr(self, field_name).extend(field_value) - elif field_cdescriptor.cpp_type == _CPPTYPE_MESSAGE: - getattr(self, field_name).MergeFrom(field_value) - else: - setattr(self, field_name, field_value) - - Init.__module__ = None - Init.__doc__ = None - cls.__init__ = Init - - -def _IsMessageSetExtension(field): - """Checks if a field is a message set extension.""" - return (field.is_extension and - field.containing_type.has_options and - field.containing_type.GetOptions().message_set_wire_format and - field.type == _TYPE_MESSAGE and - field.message_type == field.extension_scope and - field.label == _LABEL_OPTIONAL) - - -def _AddMessageMethods(message_descriptor, cls): - """Adds the methods to a protocol message class.""" - if message_descriptor.is_extendable: - - def ClearExtension(self, extension): - self.Extensions.ClearExtension(extension) - - def HasExtension(self, extension): - return self.Extensions.HasExtension(extension) - - def HasField(self, field_name): - return self._cmsg.HasField(field_name) - - def ClearField(self, field_name): - child_cmessage = None - if field_name in self._composite_fields: - child_field = self._composite_fields[field_name] - del self._composite_fields[field_name] - - child_cdescriptor = self.__descriptors[field_name] - # TODO(anuraag): Support clearing repeated message fields as well. - if (child_cdescriptor.label != _LABEL_REPEATED and - child_cdescriptor.cpp_type == _CPPTYPE_MESSAGE): - child_field._owner = None - child_cmessage = child_field._cmsg - - if child_cmessage is not None: - self._cmsg.ClearField(field_name, child_cmessage) - else: - self._cmsg.ClearField(field_name) - - def Clear(self): - cmessages_to_release = [] - for field_name, child_field in self._composite_fields.iteritems(): - child_cdescriptor = self.__descriptors[field_name] - # TODO(anuraag): Support clearing repeated message fields as well. - if (child_cdescriptor.label != _LABEL_REPEATED and - child_cdescriptor.cpp_type == _CPPTYPE_MESSAGE): - child_field._owner = None - cmessages_to_release.append((child_cdescriptor, child_field._cmsg)) - self._composite_fields.clear() - self._cmsg.Clear(cmessages_to_release) - - def IsInitialized(self, errors=None): - if self._cmsg.IsInitialized(): - return True - if errors is not None: - errors.extend(self.FindInitializationErrors()); - return False - - def SerializeToString(self): - if not self.IsInitialized(): - raise message.EncodeError( - 'Message %s is missing required fields: %s' % ( - self._cmsg.full_name, ','.join(self.FindInitializationErrors()))) - return self._cmsg.SerializeToString() - - def SerializePartialToString(self): - return self._cmsg.SerializePartialToString() - - def ParseFromString(self, serialized): - self.Clear() - self.MergeFromString(serialized) - - def MergeFromString(self, serialized): - byte_size = self._cmsg.MergeFromString(serialized) - if byte_size < 0: - raise message.DecodeError('Unable to merge from string.') - return byte_size - - def MergeFrom(self, msg): - if not isinstance(msg, cls): - raise TypeError( - "Parameter to MergeFrom() must be instance of same class: " - "expected %s got %s." % (cls.__name__, type(msg).__name__)) - self._cmsg.MergeFrom(msg._cmsg) - - def CopyFrom(self, msg): - self._cmsg.CopyFrom(msg._cmsg) - - def ByteSize(self): - return self._cmsg.ByteSize() - - def SetInParent(self): - return self._cmsg.SetInParent() - - def ListFields(self): - all_fields = [] - field_list = self._cmsg.ListFields() - fields_by_name = cls.DESCRIPTOR.fields_by_name - for is_extension, field_name in field_list: - if is_extension: - extension = cls._extensions_by_name[field_name] - all_fields.append((extension, self.Extensions[extension])) - else: - field_descriptor = fields_by_name[field_name] - all_fields.append( - (field_descriptor, getattr(self, field_name))) - all_fields.sort(key=lambda item: item[0].number) - return all_fields - - def FindInitializationErrors(self): - return self._cmsg.FindInitializationErrors() - - def __str__(self): - return str(self._cmsg) - - def __eq__(self, other): - if self is other: - return True - if not isinstance(other, self.__class__): - return False - return self.ListFields() == other.ListFields() - - def __ne__(self, other): - return not self == other - - def __hash__(self): - raise TypeError('unhashable object') - - def __unicode__(self): - # Lazy import to prevent circular import when text_format imports this file. - from google.protobuf import text_format - return text_format.MessageToString(self, as_utf8=True).decode('utf-8') - - # Attach the local methods to the message class. - for key, value in locals().copy().iteritems(): - if key not in ('key', 'value', '__builtins__', '__name__', '__doc__'): - setattr(cls, key, value) - - # Static methods: - - def RegisterExtension(extension_handle): - extension_handle.containing_type = cls.DESCRIPTOR - cls._extensions_by_name[extension_handle.full_name] = extension_handle - - if _IsMessageSetExtension(extension_handle): - # MessageSet extension. Also register under type name. - cls._extensions_by_name[ - extension_handle.message_type.full_name] = extension_handle - cls.RegisterExtension = staticmethod(RegisterExtension) - - def FromString(string): - msg = cls() - msg.MergeFromString(string) - return msg - cls.FromString = staticmethod(FromString) - - - -def _AddPropertiesForExtensions(message_descriptor, cls): - """Adds properties for all fields in this protocol message type.""" - extension_dict = message_descriptor.extensions_by_name - for extension_name, extension_field in extension_dict.iteritems(): - constant_name = extension_name.upper() + '_FIELD_NUMBER' - setattr(cls, constant_name, extension_field.number) diff --git a/python/google/protobuf/internal/descriptor_database_test.py b/python/google/protobuf/internal/descriptor_database_test.py index 8970f5c2..edf66a63 100644 --- a/python/google/protobuf/internal/descriptor_database_test.py +++ b/python/google/protobuf/internal/descriptor_database_test.py @@ -34,13 +34,14 @@ __author__ = 'matthewtoia@google.com (Matt Toia)' -from google.apputils import basetest +import unittest + from google.protobuf import descriptor_pb2 from google.protobuf.internal import factory_test2_pb2 from google.protobuf import descriptor_database -class DescriptorDatabaseTest(basetest.TestCase): +class DescriptorDatabaseTest(unittest.TestCase): def testAdd(self): db = descriptor_database.DescriptorDatabase() @@ -62,4 +63,4 @@ class DescriptorDatabaseTest(basetest.TestCase): 'google.protobuf.python.internal.MessageWithNestedEnumOnly.NestedEnum')) if __name__ == '__main__': - basetest.main() + unittest.main() diff --git a/python/google/protobuf/internal/descriptor_pool_test.py b/python/google/protobuf/internal/descriptor_pool_test.py index 11ef61c5..6d04ebaa 100644 --- a/python/google/protobuf/internal/descriptor_pool_test.py +++ b/python/google/protobuf/internal/descriptor_pool_test.py @@ -37,7 +37,6 @@ __author__ = 'matthewtoia@google.com (Matt Toia)' import os import unittest -from google.apputils import basetest from google.protobuf import unittest_pb2 from google.protobuf import descriptor_pb2 from google.protobuf.internal import api_implementation @@ -51,7 +50,7 @@ from google.protobuf import descriptor_pool from google.protobuf import symbol_database -class DescriptorPoolTest(basetest.TestCase): +class DescriptorPoolTest(unittest.TestCase): def setUp(self): self.pool = descriptor_pool.DescriptorPool() @@ -426,7 +425,7 @@ class ExtensionField(object): test.assertEqual(self.extended_type, field_desc.containing_type.name) -class AddDescriptorTest(basetest.TestCase): +class AddDescriptorTest(unittest.TestCase): def _TestMessage(self, prefix): pool = descriptor_pool.DescriptorPool() @@ -588,4 +587,4 @@ TEST2_FILE = ProtoFile( if __name__ == '__main__': - basetest.main() + unittest.main() diff --git a/python/google/protobuf/internal/descriptor_python_test.py b/python/google/protobuf/internal/descriptor_python_test.py deleted file mode 100644 index 5471ae02..00000000 --- a/python/google/protobuf/internal/descriptor_python_test.py +++ /dev/null @@ -1,54 +0,0 @@ -#! /usr/bin/python -# -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Unittest for descriptor.py for the pure Python implementation.""" - -import os -os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'python' - -# We must set the implementation version above before the google3 imports. -# pylint: disable=g-import-not-at-top -from google.apputils import basetest -from google.protobuf.internal import api_implementation -# Run all tests from the original module by putting them in our namespace. -# pylint: disable=wildcard-import -from google.protobuf.internal.descriptor_test import * - - -class ConfirmPurePythonTest(basetest.TestCase): - - def testImplementationSetting(self): - self.assertEqual('python', api_implementation.Type()) - - -if __name__ == '__main__': - basetest.main() diff --git a/python/google/protobuf/internal/descriptor_test.py b/python/google/protobuf/internal/descriptor_test.py index 50c4dbba..549af088 100755 --- a/python/google/protobuf/internal/descriptor_test.py +++ b/python/google/protobuf/internal/descriptor_test.py @@ -35,8 +35,8 @@ __author__ = 'robinson@google.com (Will Robinson)' import sys +import unittest -from google.apputils import basetest from google.protobuf import unittest_custom_options_pb2 from google.protobuf import unittest_import_pb2 from google.protobuf import unittest_pb2 @@ -52,7 +52,7 @@ name: 'TestEmptyMessage' """ -class DescriptorTest(basetest.TestCase): +class DescriptorTest(unittest.TestCase): def setUp(self): file_proto = descriptor_pb2.FileDescriptorProto( @@ -390,7 +390,7 @@ class DescriptorTest(basetest.TestCase): self.assertEqual(self.my_file.name, 'some/filename/some.proto') self.assertEqual(self.my_file.package, 'protobuf_unittest') - @basetest.unittest.skipIf( + @unittest.skipIf( api_implementation.Type() != 'cpp' or api_implementation.Version() != 2, 'Immutability of descriptors is only enforced in v2 implementation') def testImmutableCppDescriptor(self): @@ -403,7 +403,7 @@ class DescriptorTest(basetest.TestCase): message_descriptor.fields.append(None) -class GeneratedDescriptorTest(basetest.TestCase): +class GeneratedDescriptorTest(unittest.TestCase): """Tests for the properties of descriptors in generated code.""" def CheckMessageDescriptor(self, message_descriptor): @@ -493,7 +493,7 @@ class GeneratedDescriptorTest(basetest.TestCase): self.assertEqual('FOO', next(values_iter).name) -class DescriptorCopyToProtoTest(basetest.TestCase): +class DescriptorCopyToProtoTest(unittest.TestCase): """Tests for CopyTo functions of Descriptor.""" def _AssertProtoEqual(self, actual_proto, expected_class, expected_ascii): @@ -694,7 +694,7 @@ class DescriptorCopyToProtoTest(basetest.TestCase): # TEST_SERVICE_ASCII) -class MakeDescriptorTest(basetest.TestCase): +class MakeDescriptorTest(unittest.TestCase): def testMakeDescriptorWithNestedFields(self): file_descriptor_proto = descriptor_pb2.FileDescriptorProto() @@ -776,4 +776,4 @@ class MakeDescriptorTest(basetest.TestCase): options.Extensions[unittest_custom_options_pb2.msgopt].i) if __name__ == '__main__': - basetest.main() + unittest.main() diff --git a/python/google/protobuf/internal/generator_test.py b/python/google/protobuf/internal/generator_test.py index 03361e66..50fc1922 100755 --- a/python/google/protobuf/internal/generator_test.py +++ b/python/google/protobuf/internal/generator_test.py @@ -41,7 +41,8 @@ further ensures that we can use Python protocol message objects as we expect. __author__ = 'robinson@google.com (Will Robinson)' -from google.apputils import basetest +import unittest + from google.protobuf.internal import test_bad_identifiers_pb2 from google.protobuf import unittest_custom_options_pb2 from google.protobuf import unittest_import_pb2 @@ -55,7 +56,7 @@ from google.protobuf import symbol_database MAX_EXTENSION = 536870912 -class GeneratorTest(basetest.TestCase): +class GeneratorTest(unittest.TestCase): def testNestedMessageDescriptor(self): field_name = 'optional_nested_message' @@ -291,7 +292,7 @@ class GeneratorTest(basetest.TestCase): self.assertIs(desc.oneofs[0], desc.oneofs_by_name['oneof_field']) nested_names = set(['oneof_uint32', 'oneof_nested_message', 'oneof_string', 'oneof_bytes']) - self.assertSameElements( + self.assertItemsEqual( nested_names, [field.name for field in desc.oneofs[0].fields]) for field_name, field_desc in desc.fields_by_name.iteritems(): @@ -301,7 +302,7 @@ class GeneratorTest(basetest.TestCase): self.assertIsNone(field_desc.containing_oneof) -class SymbolDatabaseRegistrationTest(basetest.TestCase): +class SymbolDatabaseRegistrationTest(unittest.TestCase): """Checks that messages, enums and files are correctly registered.""" def testGetSymbol(self): @@ -340,4 +341,4 @@ class SymbolDatabaseRegistrationTest(basetest.TestCase): 'google/protobuf/unittest.proto').name) if __name__ == '__main__': - basetest.main() + unittest.main() diff --git a/python/google/protobuf/internal/message_factory_python_test.py b/python/google/protobuf/internal/message_factory_python_test.py deleted file mode 100644 index 85e02b25..00000000 --- a/python/google/protobuf/internal/message_factory_python_test.py +++ /dev/null @@ -1,54 +0,0 @@ -#! /usr/bin/python -# -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Tests for ..public.message_factory for the pure Python implementation.""" - -import os -os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'python' - -# We must set the implementation version above before the google3 imports. -# pylint: disable=g-import-not-at-top -from google.apputils import basetest -from google.protobuf.internal import api_implementation -# Run all tests from the original module by putting them in our namespace. -# pylint: disable=wildcard-import -from google.protobuf.internal.message_factory_test import * - - -class ConfirmPurePythonTest(basetest.TestCase): - - def testImplementationSetting(self): - self.assertEqual('python', api_implementation.Type()) - - -if __name__ == '__main__': - basetest.main() diff --git a/python/google/protobuf/internal/message_factory_test.py b/python/google/protobuf/internal/message_factory_test.py index fcf13410..45ddcd26 100644 --- a/python/google/protobuf/internal/message_factory_test.py +++ b/python/google/protobuf/internal/message_factory_test.py @@ -34,7 +34,8 @@ __author__ = 'matthewtoia@google.com (Matt Toia)' -from google.apputils import basetest +import unittest + from google.protobuf import descriptor_pb2 from google.protobuf.internal import factory_test1_pb2 from google.protobuf.internal import factory_test2_pb2 @@ -43,7 +44,7 @@ from google.protobuf import descriptor_pool from google.protobuf import message_factory -class MessageFactoryTest(basetest.TestCase): +class MessageFactoryTest(unittest.TestCase): def setUp(self): self.factory_test1_fd = descriptor_pb2.FileDescriptorProto.FromString( @@ -104,17 +105,18 @@ class MessageFactoryTest(basetest.TestCase): for _ in range(2): messages = message_factory.GetMessages([self.factory_test2_fd, self.factory_test1_fd]) - self.assertContainsSubset( - ['google.protobuf.python.internal.Factory2Message', - 'google.protobuf.python.internal.Factory1Message'], - messages.keys()) + self.assertTrue( + set(['google.protobuf.python.internal.Factory2Message', + 'google.protobuf.python.internal.Factory1Message'], + ).issubset(set(messages.keys()))) self._ExerciseDynamicClass( messages['google.protobuf.python.internal.Factory2Message']) - self.assertContainsSubset( - ['google.protobuf.python.internal.Factory2Message.one_more_field', - 'google.protobuf.python.internal.another_field'], - (messages['google.protobuf.python.internal.Factory1Message'] - ._extensions_by_name.keys())) + self.assertTrue( + set(['google.protobuf.python.internal.Factory2Message.one_more_field', + 'google.protobuf.python.internal.another_field'], + ).issubset( + set(messages['google.protobuf.python.internal.Factory1Message'] + ._extensions_by_name.keys()))) factory_msg1 = messages['google.protobuf.python.internal.Factory1Message'] msg1 = messages['google.protobuf.python.internal.Factory1Message']() ext1 = factory_msg1._extensions_by_name[ @@ -128,4 +130,4 @@ class MessageFactoryTest(basetest.TestCase): if __name__ == '__main__': - basetest.main() + unittest.main() diff --git a/python/google/protobuf/internal/message_python_test.py b/python/google/protobuf/internal/message_python_test.py deleted file mode 100644 index c40623a8..00000000 --- a/python/google/protobuf/internal/message_python_test.py +++ /dev/null @@ -1,54 +0,0 @@ -#! /usr/bin/python -# -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Tests for ..public.message for the pure Python implementation.""" - -import os -os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'python' - -# We must set the implementation version above before the google3 imports. -# pylint: disable=g-import-not-at-top -from google.apputils import basetest -from google.protobuf.internal import api_implementation -# Run all tests from the original module by putting them in our namespace. -# pylint: disable=wildcard-import -from google.protobuf.internal.message_test import * - - -class ConfirmPurePythonTest(basetest.TestCase): - - def testImplementationSetting(self): - self.assertEqual('python', api_implementation.Type()) - - -if __name__ == '__main__': - basetest.main() diff --git a/python/google/protobuf/internal/message_test.py b/python/google/protobuf/internal/message_test.py index ed1298af..ce79b607 100755 --- a/python/google/protobuf/internal/message_test.py +++ b/python/google/protobuf/internal/message_test.py @@ -50,7 +50,6 @@ import pickle import sys import unittest -from google.apputils import basetest from google.protobuf.internal import _parameterized from google.protobuf import unittest_pb2 from google.protobuf import unittest_proto3_arena_pb2 @@ -75,7 +74,7 @@ def IsNegInf(val): @_parameterized.Parameters( (unittest_pb2), (unittest_proto3_arena_pb2)) -class MessageTest(basetest.TestCase): +class MessageTest(unittest.TestCase): def testBadUtf8String(self, message_module): if api_implementation.Type() != 'python': @@ -887,7 +886,7 @@ class MessageTest(basetest.TestCase): # Class to test proto2-only features (required, extensions, etc.) -class Proto2Test(basetest.TestCase): +class Proto2Test(unittest.TestCase): def testFieldPresence(self): message = unittest_pb2.TestAllTypes() @@ -1037,7 +1036,7 @@ class Proto2Test(basetest.TestCase): # Class to test proto3-only features/behavior (updated field presence & enums) -class Proto3Test(basetest.TestCase): +class Proto3Test(unittest.TestCase): def testFieldPresence(self): message = unittest_proto3_arena_pb2.TestAllTypes() @@ -1115,7 +1114,7 @@ class Proto3Test(basetest.TestCase): self.assertEqual(7654321, m2.repeated_nested_enum[0]) -class ValidTypeNamesTest(basetest.TestCase): +class ValidTypeNamesTest(unittest.TestCase): def assertImportFromName(self, msg, base_name): # Parse <type 'module.class_name'> to extra 'some.name' as a string. @@ -1138,4 +1137,4 @@ class ValidTypeNamesTest(basetest.TestCase): if __name__ == '__main__': - basetest.main() + unittest.main() diff --git a/python/google/protobuf/internal/proto_builder_test.py b/python/google/protobuf/internal/proto_builder_test.py index c74db7e7..9229205a 100644 --- a/python/google/protobuf/internal/proto_builder_test.py +++ b/python/google/protobuf/internal/proto_builder_test.py @@ -32,7 +32,7 @@ """Tests for google.protobuf.proto_builder.""" -from google.apputils import basetest +import unittest from google.protobuf import descriptor_pb2 from google.protobuf import descriptor_pool @@ -40,7 +40,7 @@ from google.protobuf import proto_builder from google.protobuf import text_format -class ProtoBuilderTest(basetest.TestCase): +class ProtoBuilderTest(unittest.TestCase): def setUp(self): self._fields = { @@ -74,4 +74,4 @@ class ProtoBuilderTest(basetest.TestCase): if __name__ == '__main__': - basetest.main() + unittest.main() diff --git a/python/google/protobuf/internal/python_message.py b/python/google/protobuf/internal/python_message.py index 6ad0f90d..54f584ae 100755 --- a/python/google/protobuf/internal/python_message.py +++ b/python/google/protobuf/internal/python_message.py @@ -324,7 +324,7 @@ def _ReraiseTypeErrorWithFieldName(message_name, field_name): exc = TypeError('%s for field %s.%s' % (str(exc), message_name, field_name)) # re-raise possibly-amended exception with original traceback: - raise type(exc), exc, sys.exc_info()[2] + raise type(exc)(exc, sys.exc_info()[2]) def _AddInitMethod(message_descriptor, cls): @@ -894,7 +894,7 @@ def _AddMergeFromStringMethod(message_descriptor, cls): except (IndexError, TypeError): # Now ord(buf[p:p+1]) == ord('') gets TypeError. raise message_mod.DecodeError('Truncated message.') - except struct.error, e: + except struct.error as e: raise message_mod.DecodeError(e) return length # Return this for legacy reasons. cls.MergeFromString = MergeFromString diff --git a/python/google/protobuf/internal/reflection_test.py b/python/google/protobuf/internal/reflection_test.py index a62d9845..8f28f4d9 100755 --- a/python/google/protobuf/internal/reflection_test.py +++ b/python/google/protobuf/internal/reflection_test.py @@ -39,8 +39,8 @@ import copy import gc import operator import struct +import unittest -from google.apputils import basetest from google.protobuf import unittest_import_pb2 from google.protobuf import unittest_mset_pb2 from google.protobuf import unittest_pb2 @@ -102,7 +102,7 @@ class _MiniDecoder(object): return self._pos == len(self._bytes) -class ReflectionTest(basetest.TestCase): +class ReflectionTest(unittest.TestCase): def assertListsEqual(self, values, others): self.assertEqual(len(values), len(others)) @@ -1619,7 +1619,7 @@ class ReflectionTest(basetest.TestCase): self.assertFalse(proto.IsInitialized(errors)) self.assertEqual(errors, ['a', 'b', 'c']) - @basetest.unittest.skipIf( + @unittest.skipIf( api_implementation.Type() != 'cpp' or api_implementation.Version() != 2, 'Errors are only available from the most recent C++ implementation.') def testFileDescriptorErrors(self): @@ -1792,7 +1792,7 @@ class ReflectionTest(basetest.TestCase): # Just check the default value. self.assertEqual(57, msg.inner.value) - @basetest.unittest.skipIf( + @unittest.skipIf( api_implementation.Type() != 'cpp' or api_implementation.Version() != 2, 'CPPv2-specific test') def testBadArguments(self): @@ -1808,7 +1808,7 @@ class ReflectionTest(basetest.TestCase): # into separate TestCase classes. -class TestAllTypesEqualityTest(basetest.TestCase): +class TestAllTypesEqualityTest(unittest.TestCase): def setUp(self): self.first_proto = unittest_pb2.TestAllTypes() @@ -1824,7 +1824,7 @@ class TestAllTypesEqualityTest(basetest.TestCase): self.assertEqual(self.first_proto, self.second_proto) -class FullProtosEqualityTest(basetest.TestCase): +class FullProtosEqualityTest(unittest.TestCase): """Equality tests using completely-full protos as a starting point.""" @@ -1910,7 +1910,7 @@ class FullProtosEqualityTest(basetest.TestCase): self.assertEqual(self.first_proto, self.second_proto) -class ExtensionEqualityTest(basetest.TestCase): +class ExtensionEqualityTest(unittest.TestCase): def testExtensionEquality(self): first_proto = unittest_pb2.TestAllExtensions() @@ -1943,7 +1943,7 @@ class ExtensionEqualityTest(basetest.TestCase): self.assertEqual(first_proto, second_proto) -class MutualRecursionEqualityTest(basetest.TestCase): +class MutualRecursionEqualityTest(unittest.TestCase): def testEqualityWithMutualRecursion(self): first_proto = unittest_pb2.TestMutualRecursionA() @@ -1955,7 +1955,7 @@ class MutualRecursionEqualityTest(basetest.TestCase): self.assertEqual(first_proto, second_proto) -class ByteSizeTest(basetest.TestCase): +class ByteSizeTest(unittest.TestCase): def setUp(self): self.proto = unittest_pb2.TestAllTypes() @@ -2251,7 +2251,7 @@ class ByteSizeTest(basetest.TestCase): # * Handling of empty submessages (with and without "has" # bits set). -class SerializationTest(basetest.TestCase): +class SerializationTest(unittest.TestCase): def testSerializeEmtpyMessage(self): first_proto = unittest_pb2.TestAllTypes() @@ -2802,7 +2802,7 @@ class SerializationTest(basetest.TestCase): self.assertEqual(3, proto.repeated_int32[2]) -class OptionsTest(basetest.TestCase): +class OptionsTest(unittest.TestCase): def testMessageOptions(self): proto = unittest_mset_pb2.TestMessageSet() @@ -2829,9 +2829,9 @@ class OptionsTest(basetest.TestCase): -class ClassAPITest(basetest.TestCase): +class ClassAPITest(unittest.TestCase): - @basetest.unittest.skipIf( + @unittest.skipIf( api_implementation.Type() == 'cpp' and api_implementation.Version() == 2, 'C++ implementation requires a call to MakeDescriptor()') def testMakeClassWithNestedDescriptor(self): @@ -2963,4 +2963,4 @@ class ClassAPITest(basetest.TestCase): self.assertEqual(msg.bar.baz.deep, 4) if __name__ == '__main__': - basetest.main() + unittest.main() diff --git a/python/google/protobuf/internal/service_reflection_test.py b/python/google/protobuf/internal/service_reflection_test.py index d066ae70..e3f71545 100755 --- a/python/google/protobuf/internal/service_reflection_test.py +++ b/python/google/protobuf/internal/service_reflection_test.py @@ -34,13 +34,14 @@ __author__ = 'petar@google.com (Petar Petrov)' -from google.apputils import basetest +import unittest + from google.protobuf import unittest_pb2 from google.protobuf import service_reflection from google.protobuf import service -class FooUnitTest(basetest.TestCase): +class FooUnitTest(unittest.TestCase): def testService(self): class MockRpcChannel(service.RpcChannel): @@ -133,4 +134,4 @@ class FooUnitTest(basetest.TestCase): if __name__ == '__main__': - basetest.main() + unittest.main() diff --git a/python/google/protobuf/internal/symbol_database_test.py b/python/google/protobuf/internal/symbol_database_test.py index 47572d58..a58cb1a4 100644 --- a/python/google/protobuf/internal/symbol_database_test.py +++ b/python/google/protobuf/internal/symbol_database_test.py @@ -32,12 +32,13 @@ """Tests for google.protobuf.symbol_database.""" -from google.apputils import basetest +import unittest + from google.protobuf import unittest_pb2 from google.protobuf import symbol_database -class SymbolDatabaseTest(basetest.TestCase): +class SymbolDatabaseTest(unittest.TestCase): def _Database(self): db = symbol_database.SymbolDatabase() @@ -117,4 +118,4 @@ class SymbolDatabaseTest(basetest.TestCase): if __name__ == '__main__': - basetest.main() + unittest.main() diff --git a/python/google/protobuf/internal/text_encoding_test.py b/python/google/protobuf/internal/text_encoding_test.py index db0222bd..48c30f01 100755 --- a/python/google/protobuf/internal/text_encoding_test.py +++ b/python/google/protobuf/internal/text_encoding_test.py @@ -32,7 +32,8 @@ """Tests for google.protobuf.text_encoding.""" -from google.apputils import basetest +import unittest + from google.protobuf import text_encoding TEST_VALUES = [ @@ -50,7 +51,7 @@ TEST_VALUES = [ b"\010\011\012\013\014\015")] -class TextEncodingTestCase(basetest.TestCase): +class TextEncodingTestCase(unittest.TestCase): def testCEscape(self): for escaped, escaped_utf8, unescaped in TEST_VALUES: self.assertEquals(escaped, @@ -65,4 +66,4 @@ class TextEncodingTestCase(basetest.TestCase): if __name__ == "__main__": - basetest.main() + unittest.main() diff --git a/python/google/protobuf/internal/text_format_test.py b/python/google/protobuf/internal/text_format_test.py index 7d5813fb..0135099d 100755 --- a/python/google/protobuf/internal/text_format_test.py +++ b/python/google/protobuf/internal/text_format_test.py @@ -35,8 +35,8 @@ __author__ = 'kenton@google.com (Kenton Varda)' import re +import unittest -from google.apputils import basetest from google.protobuf.internal import _parameterized from google.protobuf import unittest_mset_pb2 @@ -47,7 +47,7 @@ from google.protobuf.internal import test_util from google.protobuf import text_format # Base class with some common functionality. -class TextFormatBase(basetest.TestCase): +class TextFormatBase(unittest.TestCase): def ReadGolden(self, golden_filename): with test_util.GoldenFile(golden_filename) as f: @@ -312,12 +312,12 @@ class TextFormatTest(TextFormatBase): def testParseGroupNotClosed(self, message_module): message = message_module.TestAllTypes() text = 'RepeatedGroup: <' - self.assertRaisesWithLiteralMatch( + self.assertRaisesRegexp( text_format.ParseError, '1:16 : Expected ">".', text_format.Parse, text, message) text = 'RepeatedGroup: {' - self.assertRaisesWithLiteralMatch( + self.assertRaisesRegexp( text_format.ParseError, '1:16 : Expected "}".', text_format.Parse, text, message) @@ -354,7 +354,7 @@ class TextFormatTest(TextFormatBase): def testParseBadIntValue(self, message_module): message = message_module.TestAllTypes() text = 'optional_int32: bork' - self.assertRaisesWithLiteralMatch( + self.assertRaisesRegexp( text_format.ParseError, ('1:17 : Couldn\'t parse integer: bork'), text_format.Parse, text, message) @@ -569,12 +569,12 @@ class Proto2Tests(TextFormatBase): def testParseBadExtension(self): message = unittest_pb2.TestAllExtensions() text = '[unknown_extension]: 8\n' - self.assertRaisesWithLiteralMatch( + self.assertRaisesRegexp( text_format.ParseError, '1:2 : Extension "unknown_extension" not registered.', text_format.Parse, text, message) message = unittest_pb2.TestAllTypes() - self.assertRaisesWithLiteralMatch( + self.assertRaisesRegexp( text_format.ParseError, ('1:2 : Message type "protobuf_unittest.TestAllTypes" does not have ' 'extensions.'), @@ -593,7 +593,7 @@ class Proto2Tests(TextFormatBase): message = unittest_pb2.TestAllExtensions() text = ('[protobuf_unittest.optional_int32_extension]: 42 ' '[protobuf_unittest.optional_int32_extension]: 67') - self.assertRaisesWithLiteralMatch( + self.assertRaisesRegexp( text_format.ParseError, ('1:96 : Message type "protobuf_unittest.TestAllExtensions" ' 'should not have multiple ' @@ -604,7 +604,7 @@ class Proto2Tests(TextFormatBase): message = unittest_pb2.TestAllTypes() text = ('optional_nested_message { bb: 1 } ' 'optional_nested_message { bb: 2 }') - self.assertRaisesWithLiteralMatch( + self.assertRaisesRegexp( text_format.ParseError, ('1:65 : Message type "protobuf_unittest.TestAllTypes.NestedMessage" ' 'should not have multiple "bb" fields.'), @@ -614,14 +614,14 @@ class Proto2Tests(TextFormatBase): message = unittest_pb2.TestAllTypes() text = ('optional_int32: 42 ' 'optional_int32: 67') - self.assertRaisesWithLiteralMatch( + self.assertRaisesRegexp( text_format.ParseError, ('1:36 : Message type "protobuf_unittest.TestAllTypes" should not ' 'have multiple "optional_int32" fields.'), text_format.Parse, text, message) -class TokenizerTest(basetest.TestCase): +class TokenizerTest(unittest.TestCase): def testSimpleTokenCases(self): text = ('identifier1:"string1"\n \n\n' @@ -766,4 +766,4 @@ class TokenizerTest(basetest.TestCase): if __name__ == '__main__': - basetest.main() + unittest.main() diff --git a/python/google/protobuf/internal/unknown_fields_test.py b/python/google/protobuf/internal/unknown_fields_test.py index 59f9ae4c..d116920e 100755 --- a/python/google/protobuf/internal/unknown_fields_test.py +++ b/python/google/protobuf/internal/unknown_fields_test.py @@ -35,7 +35,8 @@ __author__ = 'bohdank@google.com (Bohdan Koval)' -from google.apputils import basetest +import unittest + from google.protobuf import unittest_mset_pb2 from google.protobuf import unittest_pb2 from google.protobuf import unittest_proto3_arena_pb2 @@ -46,7 +47,7 @@ from google.protobuf.internal import test_util from google.protobuf.internal import type_checkers -class UnknownFieldsTest(basetest.TestCase): +class UnknownFieldsTest(unittest.TestCase): def setUp(self): self.descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR @@ -104,7 +105,7 @@ class UnknownFieldsTest(basetest.TestCase): # fields when checking equality. # # TODO(haberman): fix this. - @basetest.unittest.skipIf( + @unittest.skipIf( api_implementation.Type() == 'cpp' and api_implementation.Version() == 2, 'C++ implementation does not expose unknown fields to Python') def testEquals(self): @@ -117,10 +118,10 @@ class UnknownFieldsTest(basetest.TestCase): self.assertNotEqual(self.empty_message, message) -@basetest.unittest.skipIf( +@unittest.skipIf( api_implementation.Type() == 'cpp' and api_implementation.Version() == 2, 'C++ implementation does not expose unknown fields to Python') -class UnknownFieldsAccessorsTest(basetest.TestCase): +class UnknownFieldsAccessorsTest(unittest.TestCase): def setUp(self): self.descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR @@ -205,10 +206,10 @@ class UnknownFieldsAccessorsTest(basetest.TestCase): -@basetest.unittest.skipIf( +@unittest.skipIf( api_implementation.Type() == 'cpp' and api_implementation.Version() == 2, 'C++ implementation does not expose unknown fields to Python') -class UnknownEnumValuesTest(basetest.TestCase): +class UnknownEnumValuesTest(unittest.TestCase): def setUp(self): self.descriptor = missing_enum_values_pb2.TestEnumValues.DESCRIPTOR @@ -261,4 +262,4 @@ class UnknownEnumValuesTest(basetest.TestCase): if __name__ == '__main__': - basetest.main() + unittest.main() diff --git a/python/google/protobuf/internal/wire_format_test.py b/python/google/protobuf/internal/wire_format_test.py index f39035ca..e40a40cc 100755 --- a/python/google/protobuf/internal/wire_format_test.py +++ b/python/google/protobuf/internal/wire_format_test.py @@ -34,12 +34,13 @@ __author__ = 'robinson@google.com (Will Robinson)' -from google.apputils import basetest +import unittest + from google.protobuf import message from google.protobuf.internal import wire_format -class WireFormatTest(basetest.TestCase): +class WireFormatTest(unittest.TestCase): def testPackTag(self): field_number = 0xabc @@ -250,4 +251,4 @@ class WireFormatTest(basetest.TestCase): if __name__ == '__main__': - basetest.main() + unittest.main() diff --git a/python/setup.py b/python/setup.py index c18818e2..fa98cd31 100755 --- a/python/setup.py +++ b/python/setup.py @@ -155,7 +155,7 @@ if __name__ == '__main__': "google/protobuf/pyext/repeated_scalar_container.cc", "google/protobuf/pyext/repeated_composite_container.cc" ], define_macros=[('GOOGLE_PROTOBUF_HAS_ONEOF', '1')], - include_dirs = [ ".", "..", "../src"], + include_dirs = [ ".", "../src" ], libraries = [ "protobuf" ], library_dirs = [ '../src/.libs' ], )) @@ -165,12 +165,11 @@ if __name__ == '__main__': version = GetVersion(), packages = [ 'google' ], namespace_packages = [ 'google' ], - google_test_dir = "google/protobuf/internal", + test_suite = 'google.protobuf.internal', # Must list modules explicitly so that we don't install tests. py_modules = [ 'google.protobuf.internal.api_implementation', 'google.protobuf.internal.containers', - 'google.protobuf.internal.cpp_message', 'google.protobuf.internal.decoder', 'google.protobuf.internal.encoder', 'google.protobuf.internal.enum_type_wrapper', @@ -195,11 +194,6 @@ if __name__ == '__main__': 'google.protobuf.text_format'], cmdclass = { 'clean': clean, 'build_py': build_py }, install_requires = ['setuptools'], - # TODO: Restore dependency once a Python 3 compatible google-apputils - # is released. - setup_requires = (['google-apputils'] - if sys.version_info[0] < 3 else - []), ext_modules = ext_module_list, url = 'https://developers.google.com/protocol-buffers/', maintainer = maintainer_email, diff --git a/ruby/ext/google/protobuf_c/defs.c b/ruby/ext/google/protobuf_c/defs.c index b39c27f4..9b590a55 100644 --- a/ruby/ext/google/protobuf_c/defs.c +++ b/ruby/ext/google/protobuf_c/defs.c @@ -250,7 +250,7 @@ void Descriptor_free(void* _self) { &self->pb_serialize_handlers); } if (self->json_serialize_handlers) { - upb_handlers_unref(self->pb_serialize_handlers, + upb_handlers_unref(self->json_serialize_handlers, &self->json_serialize_handlers); } xfree(self); diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index 4c9449d2..abbbde35 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = "google-protobuf" - s.version = "3.0.0.alpha.3.0.pre" + s.version = "3.0.0.alpha.3.1.pre" s.licenses = ["BSD"] s.summary = "Protocol Buffers" s.description = "Protocol Buffers are Google's data interchange format." diff --git a/ruby/tests/basic.rb b/ruby/tests/basic.rb index 9d0f0a98..1c3fb62c 100644 --- a/ruby/tests/basic.rb +++ b/ruby/tests/basic.rb @@ -8,6 +8,19 @@ require 'test/unit' module BasicTest pool = Google::Protobuf::DescriptorPool.new pool.build do + add_message "Foo" do + optional :bar, :message, 1, "Bar" + repeated :baz, :message, 2, "Baz" + end + + add_message "Bar" do + optional :msg, :string, 1 + end + + add_message "Baz" do + optional :msg, :string, 1 + end + add_message "TestMessage" do optional :optional_int32, :int32, 1 optional :optional_int64, :int64, 2 @@ -84,6 +97,9 @@ module BasicTest end end + Foo = pool.lookup("Foo").msgclass + Bar = pool.lookup("Bar").msgclass + Baz = pool.lookup("Baz").msgclass TestMessage = pool.lookup("TestMessage").msgclass TestMessage2 = pool.lookup("TestMessage2").msgclass Recursive1 = pool.lookup("Recursive1").msgclass @@ -992,6 +1008,14 @@ module BasicTest json_text = TestMessage.encode_json(m) m2 = TestMessage.decode_json(json_text) assert m == m2 + + # Crash case from GitHub issue 283. + bar = Bar.new(msg: "bar") + baz1 = Baz.new(msg: "baz") + baz2 = Baz.new(msg: "quux") + Foo.encode_json(Foo.new) + Foo.encode_json(Foo.new(bar: bar)) + Foo.encode_json(Foo.new(bar: bar, baz: [baz1, baz2])) end def test_json_maps diff --git a/src/Makefile.am b/src/Makefile.am index 1a82448e..049583f5 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -352,7 +352,7 @@ protoc_inputs = \ google/protobuf/unittest_preserve_unknown_enum2.proto \ google/protobuf/unittest_proto3_arena.proto \ google/protobuf/compiler/cpp/cpp_test_bad_identifiers.proto \ - google/protobuf/compiler/cpp/test_large_enum_value.proto + google/protobuf/compiler/cpp/cpp_test_large_enum_value.proto EXTRA_DIST = \ $(protoc_inputs) \ @@ -428,8 +428,8 @@ protoc_outputs = \ google/protobuf/unittest_preserve_unknown_enum2.pb.h \ google/protobuf/unittest_proto3_arena.pb.cc \ google/protobuf/unittest_proto3_arena.pb.h \ - google/protobuf/compiler/cpp/test_large_enum_value.pb.cc \ - google/protobuf/compiler/cpp/test_large_enum_value.pb.h \ + google/protobuf/compiler/cpp/cpp_test_large_enum_value.pb.cc \ + google/protobuf/compiler/cpp/cpp_test_large_enum_value.pb.h \ google/protobuf/compiler/cpp/cpp_test_bad_identifiers.pb.cc \ google/protobuf/compiler/cpp/cpp_test_bad_identifiers.pb.h diff --git a/src/google/protobuf/SEBS b/src/google/protobuf/SEBS deleted file mode 100644 index ba33c732..00000000 --- a/src/google/protobuf/SEBS +++ /dev/null @@ -1,240 +0,0 @@ -# **EXPERIMENTAL** -# -# See http://sebs.googlecode.com -# -# This is an experimental build definition file using the SEBS build system. -# I (Kenton Varda, maintainer of Protocol Buffers) happen to be the author of -# SEBS, though SEBS is not a Google project. I'm sticking this file in -# protobuf's SVN because that's the easiest place for me to put it, and it -# shouldn't harm anyone. This file is not included in the distribution. -# -# Currently, to use this file, you must generate config.h and put it at the -# top level of the source tree. - -_cpp = sebs.import_("//sebs/cpp.sebs") - -# ==================================================================== -# Public targets - -protobuf_lite = _cpp.Library( - name = "protobuf-lite", - srcs = [ "stubs/common.cc", - "stubs/once.cc", - "stubs/hash.cc", - "stubs/hash.h", - "stubs/map-util.h", - "stubs/stl_util-inl.h", - "extension_set.cc", - "generated_message_util.cc", - "message_lite.cc", - "repeated_field.cc", - "wire_format_lite.cc", - "io/coded_stream.cc", - "io/zero_copy_stream.cc", - "io/zero_copy_stream_impl_lite.cc" ], - deps = [ _cpp.SystemLibrary(name = "pthread") ]) - -protobuf = _cpp.Library( - name = "protobuf", - srcs = [ "stubs/strutil.cc", - "stubs/strutil.h", - "stubs/substitute.cc", - "stubs/substitute.h", - "stubs/structurally_valid.cc", - "descriptor.cc", - "descriptor.pb.cc", - "descriptor_database.cc", - "dynamic_message.cc", - "extension_set_heavy.cc", - "generated_message_reflection.cc", - "message.cc", - "reflection_ops.cc", - "service.cc", - "text_format.cc", - "unknown_field_set.cc", - "wire_format.cc", - "io/gzip_stream.cc", - "io/printer.cc", - "io/tokenizer.cc", - "io/zero_copy_stream_impl.cc", - "compiler/importer.cc", - "compiler/parser.cc" ], - deps = [ protobuf_lite, - _cpp.SystemLibrary(name = "z") ]) - -libprotoc = _cpp.Library( - name = "protoc", - srcs = [ "compiler/code_generator.cc", - "compiler/command_line_interface.cc", - "compiler/cpp/cpp_enum.cc", - "compiler/cpp/cpp_enum.h", - "compiler/cpp/cpp_enum_field.cc", - "compiler/cpp/cpp_enum_field.h", - "compiler/cpp/cpp_extension.cc", - "compiler/cpp/cpp_extension.h", - "compiler/cpp/cpp_field.cc", - "compiler/cpp/cpp_field.h", - "compiler/cpp/cpp_file.cc", - "compiler/cpp/cpp_file.h", - "compiler/cpp/cpp_generator.cc", - "compiler/cpp/cpp_helpers.cc", - "compiler/cpp/cpp_helpers.h", - "compiler/cpp/cpp_message.cc", - "compiler/cpp/cpp_message.h", - "compiler/cpp/cpp_message_field.cc", - "compiler/cpp/cpp_message_field.h", - "compiler/cpp/cpp_primitive_field.cc", - "compiler/cpp/cpp_primitive_field.h", - "compiler/cpp/cpp_service.cc", - "compiler/cpp/cpp_service.h", - "compiler/cpp/cpp_string_field.cc", - "compiler/cpp/cpp_string_field.h", - "compiler/java/java_enum.cc", - "compiler/java/java_enum.h", - "compiler/java/java_enum_field.cc", - "compiler/java/java_enum_field.h", - "compiler/java/java_extension.cc", - "compiler/java/java_extension.h", - "compiler/java/java_field.cc", - "compiler/java/java_field.h", - "compiler/java/java_file.cc", - "compiler/java/java_file.h", - "compiler/java/java_generator.cc", - "compiler/java/java_helpers.cc", - "compiler/java/java_helpers.h", - "compiler/java/java_message.cc", - "compiler/java/java_message.h", - "compiler/java/java_message_field.cc", - "compiler/java/java_message_field.h", - "compiler/java/java_primitive_field.cc", - "compiler/java/java_primitive_field.h", - "compiler/java/java_service.cc", - "compiler/java/java_service.h", - "compiler/python/python_generator.cc" ], - deps = [ protobuf ]) - -protoc = _cpp.Binary( - name = "protoc", - srcs = [ "compiler/main.cc" ], - deps = [ libprotoc ]) - -# ==================================================================== -# ProtobufLibrary rule class - -class ProtobufLibrary(sebs.Rule): - argument_spec = sebs.ArgumentSpec(srcs = [sebs.Artifact], - deps = ([sebs.Rule], []), - lite = (bool, False)) - - def _expand(self, args): - for dep in args.deps: - if not isinstance(dep, ProtobufLibrary): - raise sebs.DefinitionError( - "Dependency of ProtobufLibrary is not a ProtobufLibrary: %s" % dep) - - protoc.expand_once() - - # We must build protoc for the host configuration to allow cross-compiling. - host_protoc = self.context.configured_artifact(protoc.binary, "host") - - protoc_action = self.context.action(self, "protobuf") - protoc_args = [host_protoc, "-Isrc", "-Itmp", "-Iinclude","--cpp_out=tmp"] - - cpp_srcs = [] - for src in args.srcs: - protoc_args.append(src) - - # We cannot build .proto files from other packages because the .pb.cc - # and .pb.h files would be written to that package, and we aren't allowed - # to write to other packages. - if self.context.local_filename(src) is None: - raise sebs.DefinitionError( - "Source file is not in this package: %s" % src) - - cc_artifact = self.context.derived_artifact(src, ".pb.cc", protoc_action) - header_artifact = self.context.derived_artifact( - src, ".pb.h", protoc_action) - - cpp_srcs.append(cc_artifact) - cpp_srcs.append(header_artifact) - - protoc_action.set_command( - sebs.SubprocessCommand(protoc_action, protoc_args, implicit = cpp_srcs)) - - deps = list(args.deps) - if args.lite: - deps.append(protobuf_lite) - else: - deps.append(protobuf) - - self.__cpp_library = _cpp.Library(srcs = cpp_srcs, deps = deps, - context = self.context) - self.__cpp_library.label = self.label - self.outputs = [] - - def as_cpp_library(self): - self.expand_once() - return self.__cpp_library - -# ==================================================================== -# Tests - -_lite_test_protos = ProtobufLibrary( - srcs = [ "unittest_lite.proto", - "unittest_import_lite.proto" ], - lite = True) -_test_protos = ProtobufLibrary( - srcs = [ "unittest.proto", - "unittest_empty.proto", - "unittest_import.proto", - "unittest_mset.proto", - "unittest_optimize_for.proto", - "unittest_embed_optimize_for.proto", - "unittest_custom_options.proto", - "unittest_lite_imports_nonlite.proto", - "compiler/cpp/cpp_test_bad_identifiers.proto" ], - deps = [ _lite_test_protos ]) - -_test_util = _cpp.Library( - name = "test_util", - srcs = [ "test_util.cc", - "test_util.h", - "testing/googletest.cc", - "testing/googletest.h", - "testing/file.cc", - "testing/file.h" ], - deps = [ protobuf, _test_protos, _cpp.SystemLibrary(name = "gtest")] ) - -protobuf_lite_test = _cpp.Test( - srcs = [ "lite_unittest.cc", - "test_util_lite.cc", - "test_util_lite.h" ], - deps = [ _lite_test_protos ]) - -protobuf_test = _cpp.Test( - srcs = [ "stubs/common_unittest.cc", - "stubs/once_unittest.cc", - "stubs/strutil_unittest.cc", - "stubs/structurally_valid_unittest.cc", - "descriptor_database_unittest.cc", - "descriptor_unittest.cc", - "dynamic_message_unittest.cc", - "extension_set_unittest.cc", - "generated_message_reflection_unittest.cc", - "message_unittest.cc", - "reflection_ops_unittest.cc", - "repeated_field_unittest.cc", - "text_format_unittest.cc", - "unknown_field_set_unittest.cc", - "wire_format_unittest.cc", - "io/coded_stream_unittest.cc", - "io/printer_unittest.cc", - "io/tokenizer_unittest.cc", - "io/zero_copy_stream_unittest.cc", - "compiler/command_line_interface_unittest.cc", - "compiler/importer_unittest.cc", - "compiler/parser_unittest.cc", - "compiler/cpp/cpp_bootstrap_unittest.cc", - "compiler/cpp/cpp_unittest.cc" ], - deps = [ protobuf, libprotoc, _test_util, - _cpp.SystemLibrary(name = "gtest_main") ]) diff --git a/src/google/protobuf/arena_nc_test.py b/src/google/protobuf/arena_nc_test.py index 41b1b5f8..fc510a48 100644 --- a/src/google/protobuf/arena_nc_test.py +++ b/src/google/protobuf/arena_nc_test.py @@ -32,11 +32,12 @@ """Negative compilation unit tests for arena API.""" +import unittest + from google3.testing.pybase import fake_target_util -from google.apputils import basetest -class ArenaNcTest(basetest.TestCase): +class ArenaNcTest(unittest.TestCase): def testCompilerErrors(self): """Runs a list of tests to verify compiler error messages.""" @@ -56,4 +57,4 @@ class ArenaNcTest(basetest.TestCase): ) if __name__ == '__main__': - basetest.main() + unittest.main() diff --git a/src/google/protobuf/compiler/command_line_interface_unittest.cc b/src/google/protobuf/compiler/command_line_interface_unittest.cc index 2b26f3be..a2cbbdc6 100644 --- a/src/google/protobuf/compiler/command_line_interface_unittest.cc +++ b/src/google/protobuf/compiler/command_line_interface_unittest.cc @@ -991,7 +991,7 @@ TEST_F(CommandLineInterfaceTest, WriteDependencyManifestFile) { " optional Foo foo = 1;\n" "}\n"); - string current_working_directory = get_current_dir_name(); + string current_working_directory = getcwd(NULL, 0); SwitchToTempDirectory(); Run("protocol_compiler --dependency_out=manifest --test_out=. " diff --git a/src/google/protobuf/compiler/cpp/test_large_enum_value.proto b/src/google/protobuf/compiler/cpp/cpp_test_large_enum_value.proto index cb6ca1b1..cb6ca1b1 100644 --- a/src/google/protobuf/compiler/cpp/test_large_enum_value.proto +++ b/src/google/protobuf/compiler/cpp/cpp_test_large_enum_value.proto |