diff options
author | temporal <temporal@630680e5-0e50-0410-840e-4b1c322b438d> | 2008-08-13 03:15:00 +0000 |
---|---|---|
committer | temporal <temporal@630680e5-0e50-0410-840e-4b1c322b438d> | 2008-08-13 03:15:00 +0000 |
commit | 779f61c6a3ce02a119e28e802f229e61b69b9046 (patch) | |
tree | 9131ef5f0acdc3d708a795fc6703488674741ee0 /src/google/protobuf/compiler | |
parent | a0f27fcd96c5bf2509ca88cca54f00b78f7b8bc5 (diff) | |
download | protobuf-779f61c6a3ce02a119e28e802f229e61b69b9046.tar.gz protobuf-779f61c6a3ce02a119e28e802f229e61b69b9046.tar.bz2 protobuf-779f61c6a3ce02a119e28e802f229e61b69b9046.zip |
Integrate recent changes from google3.
protoc
- New flags --encode and --decode can be used to convert between protobuf text
format and binary format from the command-line.
- New flag --descriptor_set_out can be used to write FileDescriptorProtos for
all parsed files directly into a single output file. This is particularly
useful if you wish to parse .proto files from programs written in languages
other than C++: just run protoc as a background process and have it output
a FileDescriptorList, then parse that natively.
C++
- Reflection objects are now per-class rather than per-instance. To make this
possible, the Reflection interface had to be changed such that all methods
take the Message instance as a parameter. This change improves performance
significantly in memory-bandwidth-limited use cases, since it makes the
message objects smaller. Note that source-incompatible interface changes
like this will not be made again after the library leaves beta.
Python
- MergeFrom(message) and CopyFrom(message) are now implemented.
- SerializeToString() raises an exception if the message is missing required
fields.
- Code organization improvements.
- Fixed doc comments for RpcController and RpcChannel, which had somehow been
swapped.
Diffstat (limited to 'src/google/protobuf/compiler')
-rw-r--r-- | src/google/protobuf/compiler/command_line_interface.cc | 309 | ||||
-rw-r--r-- | src/google/protobuf/compiler/command_line_interface.h | 28 | ||||
-rw-r--r-- | src/google/protobuf/compiler/command_line_interface_unittest.cc | 253 | ||||
-rw-r--r-- | src/google/protobuf/compiler/cpp/cpp_file.cc | 12 | ||||
-rw-r--r-- | src/google/protobuf/compiler/cpp/cpp_helpers.cc | 2 | ||||
-rw-r--r-- | src/google/protobuf/compiler/cpp/cpp_message.cc | 83 | ||||
-rw-r--r-- | src/google/protobuf/compiler/cpp/cpp_unittest.cc | 2 | ||||
-rw-r--r-- | src/google/protobuf/compiler/parser.cc | 26 |
8 files changed, 646 insertions, 69 deletions
diff --git a/src/google/protobuf/compiler/command_line_interface.cc b/src/google/protobuf/compiler/command_line_interface.cc index 8f559f55..d58fc3b8 100644 --- a/src/google/protobuf/compiler/command_line_interface.cc +++ b/src/google/protobuf/compiler/command_line_interface.cc @@ -35,6 +35,8 @@ #include <google/protobuf/compiler/importer.h> #include <google/protobuf/compiler/code_generator.h> #include <google/protobuf/descriptor.h> +#include <google/protobuf/text_format.h> +#include <google/protobuf/dynamic_message.h> #include <google/protobuf/io/zero_copy_stream_impl.h> #include <google/protobuf/stubs/common.h> #include <google/protobuf/stubs/strutil.h> @@ -52,6 +54,12 @@ namespace compiler { #ifndef F_OK #define F_OK 00 // not defined by MSVC for whatever reason #endif +#ifndef STDIN_FILENO +#define STDIN_FILENO 0 +#endif +#ifndef STDOUT_FILENO +#define STDOUT_FILENO 1 +#endif #endif #ifndef O_BINARY @@ -82,10 +90,31 @@ static bool IsWindowsAbsolutePath(const string& text) { #endif } +void SetFdToTextMode(int fd) { +#ifdef _WIN32 + if (_setmode(fd, _O_TEXT) == -1) { + // This should never happen, I think. + GOOGLE_LOG(WARNING) << "_setmode(" << fd << ", _O_TEXT): " << strerror(errno); + } +#endif + // (Text and binary are the same on non-Windows platforms.) +} + +void SetFdToBinaryMode(int fd) { +#ifdef _WIN32 + if (_setmode(fd, _O_BINARY) == -1) { + // This should never happen, I think. + GOOGLE_LOG(WARNING) << "_setmode(" << fd << ", _O_BINARY): " << strerror(errno); + } +#endif + // (Text and binary are the same on non-Windows platforms.) +} + } // namespace // A MultiFileErrorCollector that prints errors to stderr. -class CommandLineInterface::ErrorPrinter : public MultiFileErrorCollector { +class CommandLineInterface::ErrorPrinter : public MultiFileErrorCollector, + public io::ErrorCollector { public: ErrorPrinter() {} ~ErrorPrinter() {} @@ -101,6 +130,11 @@ class CommandLineInterface::ErrorPrinter : public MultiFileErrorCollector { } cerr << ": " << message << endl; } + + // implements io::ErrorCollector ----------------------------------- + void AddError(int line, int column, const string& message) { + AddError("input", line, column, message); + } }; // ------------------------------------------------------------------- @@ -243,7 +277,9 @@ CommandLineInterface::ErrorReportingFileOutput::~ErrorReportingFileOutput() { // =================================================================== CommandLineInterface::CommandLineInterface() - : disallow_services_(false), + : mode_(MODE_COMPILE), + imports_in_descriptor_set_(false), + disallow_services_(false), inputs_are_proto_path_relative_(false) {} CommandLineInterface::~CommandLineInterface() {} @@ -258,7 +294,7 @@ void CommandLineInterface::RegisterGenerator(const string& flag_name, int CommandLineInterface::Run(int argc, const char* const argv[]) { Clear(); - if (!ParseArguments(argc, argv)) return -1; + if (!ParseArguments(argc, argv)) return 1; // Set up the source tree. DiskSourceTree source_tree; @@ -269,32 +305,61 @@ int CommandLineInterface::Run(int argc, const char* const argv[]) { // Map input files to virtual paths if necessary. if (!inputs_are_proto_path_relative_) { if (!MakeInputsBeProtoPathRelative(&source_tree)) { - return -1; + return 1; } } // Allocate the Importer. ErrorPrinter error_collector; - DescriptorPool pool; Importer importer(&source_tree, &error_collector); + vector<const FileDescriptor*> parsed_files; + // Parse each file and generate output. for (int i = 0; i < input_files_.size(); i++) { // Import the file. const FileDescriptor* parsed_file = importer.Import(input_files_[i]); - if (parsed_file == NULL) return -1; + if (parsed_file == NULL) return 1; + parsed_files.push_back(parsed_file); // Enforce --disallow_services. if (disallow_services_ && parsed_file->service_count() > 0) { cerr << parsed_file->name() << ": This file contains services, but " "--disallow_services was used." << endl; - return -1; + return 1; } - // Generate output files. - for (int i = 0; i < output_directives_.size(); i++) { - if (!GenerateOutput(parsed_file, output_directives_[i])) { - return -1; + if (mode_ == MODE_COMPILE) { + // Generate output files. + for (int i = 0; i < output_directives_.size(); i++) { + if (!GenerateOutput(parsed_file, output_directives_[i])) { + return 1; + } + } + } + } + + if (!descriptor_set_name_.empty()) { + if (!WriteDescriptorSet(parsed_files)) { + return 1; + } + } + + if (mode_ == MODE_ENCODE || mode_ == MODE_DECODE) { + if (codec_type_.empty()) { + // HACK: Define an EmptyMessage type to use for decoding. + DescriptorPool pool; + FileDescriptorProto file; + file.set_name("empty_message.proto"); + file.add_message_type()->set_name("EmptyMessage"); + GOOGLE_CHECK(pool.BuildFile(file) != NULL); + codec_type_ = "EmptyMessage"; + if (!EncodeOrDecode(&pool)) { + return 1; + } + } else { + if (!EncodeOrDecode(importer.pool())) { + return 1; } } } @@ -303,9 +368,18 @@ int CommandLineInterface::Run(int argc, const char* const argv[]) { } void CommandLineInterface::Clear() { + // Clear all members that are set by Run(). Note that we must not clear + // members which are set by other methods before Run() is called. + executable_name_.clear(); proto_path_.clear(); input_files_.clear(); output_directives_.clear(); + codec_type_.clear(); + descriptor_set_name_.clear(); + + mode_ = MODE_COMPILE; + imports_in_descriptor_set_ = false; + disallow_services_ = false; } bool CommandLineInterface::MakeInputsBeProtoPathRelative( @@ -351,9 +425,12 @@ bool CommandLineInterface::ParseArguments(int argc, const char* const argv[]) { string name, value; if (ParseArgument(argv[i], &name, &value)) { - // Retured true => Use the next argument as the flag value. + // Returned true => Use the next argument as the flag value. if (i + 1 == argc || argv[i+1][0] == '-') { cerr << "Missing value for flag: " << name << endl; + if (name == "--decode") { + cerr << "To decode an unknown message, use --decode_raw." << endl; + } return false; } else { ++i; @@ -370,14 +447,23 @@ bool CommandLineInterface::ParseArguments(int argc, const char* const argv[]) { } // Check some errror cases. - if (input_files_.empty()) { + bool decoding_raw = (mode_ == MODE_DECODE) && codec_type_.empty(); + if (decoding_raw && !input_files_.empty()) { + cerr << "When using --decode_raw, no input files should be given." << endl; + return false; + } else if (!decoding_raw && input_files_.empty()) { cerr << "Missing input file." << endl; return false; } - if (output_directives_.empty()) { + if (mode_ == MODE_COMPILE && output_directives_.empty() && + descriptor_set_name_.empty()) { cerr << "Missing output directives." << endl; return false; } + if (imports_in_descriptor_set_ && descriptor_set_name_.empty()) { + cerr << "--include_imports only makes sense when combined with " + "--descriptor_set_name." << endl; + } return true; } @@ -428,7 +514,9 @@ bool CommandLineInterface::ParseArgument(const char* arg, if (*name == "-h" || *name == "--help" || *name == "--disallow_services" || - *name == "--version") { + *name == "--include_imports" || + *name == "--version" || + *name == "--decode_raw") { // HACK: These are the only flags that don't take a value. // They probably should not be hard-coded like this but for now it's // not worth doing better. @@ -487,6 +575,29 @@ bool CommandLineInterface::InterpretArgument(const string& name, proto_path_.push_back(make_pair(virtual_path, disk_path)); } + } else if (name == "-o" || name == "--descriptor_set_out") { + if (!descriptor_set_name_.empty()) { + cerr << name << " may only be passed once." << endl; + return false; + } + if (value.empty()) { + cerr << name << " requires a non-empty value." << endl; + return false; + } + if (mode_ != MODE_COMPILE) { + cerr << "Cannot use --encode or --decode and generate descriptors at the " + "same time." << endl; + return false; + } + descriptor_set_name_ = value; + + } else if (name == "--include_imports") { + if (imports_in_descriptor_set_) { + cerr << name << " may only be passed once." << endl; + return false; + } + imports_in_descriptor_set_ = true; + } else if (name == "-h" || name == "--help") { PrintHelpText(); return false; // Exit without running compiler. @@ -503,6 +614,33 @@ bool CommandLineInterface::InterpretArgument(const string& name, } else if (name == "--disallow_services") { disallow_services_ = true; + } else if (name == "--encode" || name == "--decode" || + name == "--decode_raw") { + if (mode_ != MODE_COMPILE) { + cerr << "Only one of --encode and --decode can be specified." << endl; + return false; + } + if (!output_directives_.empty() || !descriptor_set_name_.empty()) { + cerr << "Cannot use " << name + << " and generate code or descriptors at the same time." << endl; + return false; + } + + mode_ = (name == "--encode") ? MODE_ENCODE : MODE_DECODE; + + if (value.empty() && name != "--decode_raw") { + cerr << "Type name for " << name << " cannot be blank." << endl; + if (name == "--decode") { + cerr << "To decode an unknown message, use --decode_raw." << endl; + } + return false; + } else if (!value.empty() && name == "--decode_raw") { + cerr << "--decode_raw does not take a parameter." << endl; + return false; + } + + codec_type_ = value; + } else { // Some other flag. Look it up in the generators list. GeneratorMap::const_iterator iter = generators_.find(name); @@ -512,6 +650,12 @@ bool CommandLineInterface::InterpretArgument(const string& name, } // It's an output flag. Add it to the output directives. + if (mode_ != MODE_COMPILE) { + cerr << "Cannot use --encode or --decode and generate code at the " + "same time." << endl; + return false; + } + OutputDirective directive; directive.name = name; directive.generator = iter->second.generator; @@ -536,14 +680,33 @@ bool CommandLineInterface::InterpretArgument(const string& name, void CommandLineInterface::PrintHelpText() { // Sorry for indentation here; line wrapping would be uglier. cerr << -"Usage: " << executable_name_ << " [OPTION] PROTO_FILE\n" -"Parse PROTO_FILE and generate output based on the options given:\n" +"Usage: " << executable_name_ << " [OPTION] PROTO_FILES\n" +"Parse PROTO_FILES and generate output based on the options given:\n" " -IPATH, --proto_path=PATH Specify the directory in which to search for\n" " imports. May be specified multiple times;\n" " directories will be searched in order. If not\n" " given, the current working directory is used.\n" " --version Show version info and exit.\n" -" -h, --help Show this text and exit." << endl; +" -h, --help Show this text and exit.\n" +" --encode=MESSAGE_TYPE Read a text-format message of the given type\n" +" from standard input and write it in binary\n" +" to standard output. The message type must\n" +" be defined in PROTO_FILES or their imports.\n" +" --decode=MESSAGE_TYPE Read a binary message of the given type from\n" +" standard input and write it in text format\n" +" to standard output. The message type must\n" +" be defined in PROTO_FILES or their imports.\n" +" --decode_raw Read an arbitrary protocol message from\n" +" standard input and write the raw tag/value\n" +" pairs in text format to standard output. No\n" +" PROTO_FILES should be given when using this\n" +" flag.\n" +" -oFILE, Writes a FileDescriptorSet (a protocol buffer,\n" +" --descriptor_set_out=FILE defined in descriptor.proto) containing all of\n" +" the input files to FILE.\n" +" --include_imports When using --descriptor_set_out, also include\n" +" all dependencies of the input files in the\n" +" set, so that the set is self-contained." << endl; for (GeneratorMap::iterator iter = generators_.begin(); iter != generators_.end(); ++iter) { @@ -584,6 +747,116 @@ bool CommandLineInterface::GenerateOutput( return true; } +bool CommandLineInterface::EncodeOrDecode(const DescriptorPool* pool) { + // Look up the type. + const Descriptor* type = pool->FindMessageTypeByName(codec_type_); + if (type == NULL) { + cerr << "Type not defined: " << codec_type_ << endl; + return false; + } + + DynamicMessageFactory dynamic_factory(pool); + scoped_ptr<Message> message(dynamic_factory.GetPrototype(type)->New()); + + if (mode_ == MODE_ENCODE) { + SetFdToTextMode(STDIN_FILENO); + SetFdToBinaryMode(STDOUT_FILENO); + } else { + SetFdToBinaryMode(STDIN_FILENO); + SetFdToTextMode(STDOUT_FILENO); + } + + io::FileInputStream in(STDIN_FILENO); + io::FileOutputStream out(STDOUT_FILENO); + + if (mode_ == MODE_ENCODE) { + // Input is text. + ErrorPrinter error_collector; + TextFormat::Parser parser; + parser.RecordErrorsTo(&error_collector); + parser.AllowPartialMessage(true); + + if (!parser.Parse(&in, message.get())) { + cerr << "Failed to parse input." << endl; + return false; + } + } else { + // Input is binary. + if (!message->ParsePartialFromZeroCopyStream(&in)) { + cerr << "Failed to parse input." << endl; + return false; + } + } + + if (!message->IsInitialized()) { + cerr << "warning: Input message is missing required fields: " + << message->InitializationErrorString() << endl; + } + + if (mode_ == MODE_ENCODE) { + // Output is binary. + if (!message->SerializePartialToZeroCopyStream(&out)) { + cerr << "output: I/O error." << endl; + return false; + } + } else { + // Output is text. + if (!TextFormat::Print(*message, &out)) { + cerr << "output: I/O error." << endl; + return false; + } + } + + return true; +} + +bool CommandLineInterface::WriteDescriptorSet( + const vector<const FileDescriptor*> parsed_files) { + FileDescriptorSet file_set; + set<const FileDescriptor*> already_added; + vector<const FileDescriptor*> to_add(parsed_files); + + while (!to_add.empty()) { + const FileDescriptor* file = to_add.back(); + to_add.pop_back(); + if (already_added.insert(file).second) { + // This file was not already in the set. + file->CopyTo(file_set.add_file()); + + if (imports_in_descriptor_set_) { + // Add all of this file's dependencies. + for (int i = 0; i < file->dependency_count(); i++) { + to_add.push_back(file->dependency(i)); + } + } + } + } + + int fd; + do { + fd = open(descriptor_set_name_.c_str(), + O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666); + } while (fd < 0 && errno == EINTR); + + if (fd < 0) { + perror(descriptor_set_name_.c_str()); + return false; + } + + io::FileOutputStream out(fd); + if (!file_set.SerializeToZeroCopyStream(&out)) { + cerr << descriptor_set_name_ << ": " << strerror(out.GetErrno()) << endl; + out.Close(); + return false; + } + if (!out.Close()) { + cerr << descriptor_set_name_ << ": " << strerror(out.GetErrno()) << endl; + return false; + } + + return true; +} + } // namespace compiler } // namespace protobuf diff --git a/src/google/protobuf/compiler/command_line_interface.h b/src/google/protobuf/compiler/command_line_interface.h index d3cae75e..9185e47a 100644 --- a/src/google/protobuf/compiler/command_line_interface.h +++ b/src/google/protobuf/compiler/command_line_interface.h @@ -35,6 +35,7 @@ namespace google { namespace protobuf { class FileDescriptor; // descriptor.h +class DescriptorPool; // descriptor.h namespace compiler { @@ -164,6 +165,12 @@ class LIBPROTOC_EXPORT CommandLineInterface { bool GenerateOutput(const FileDescriptor* proto_file, const OutputDirective& output_directive); + // Implements --encode and --decode. + bool EncodeOrDecode(const DescriptorPool* pool); + + // Implements the --descriptor_set_out option. + bool WriteDescriptorSet(const vector<const FileDescriptor*> parsed_files); + // ----------------------------------------------------------------- // The name of the executable as invoked (i.e. argv[0]). @@ -181,6 +188,14 @@ class LIBPROTOC_EXPORT CommandLineInterface { GeneratorMap generators_; // Stuff parsed from command line. + enum Mode { + MODE_COMPILE, // Normal mode: parse .proto files and compile them. + MODE_ENCODE, // --encode: read text from stdin, write binary to stdout. + MODE_DECODE // --decode: read binary from stdin, write text to stdout. + }; + + Mode mode_; + vector<pair<string, string> > proto_path_; // Search path for proto files. vector<string> input_files_; // Names of the input proto files. @@ -194,6 +209,19 @@ class LIBPROTOC_EXPORT CommandLineInterface { }; vector<OutputDirective> output_directives_; + // When using --encode or --decode, this names the type we are encoding or + // decoding. (Empty string indicates --decode_raw.) + string codec_type_; + + // If --descriptor_set_out was given, this is the filename to which the + // FileDescriptorSet should be written. Otherwise, empty. + string descriptor_set_name_; + + // True if --include_imports was given, meaning that we should + // write all transitive dependencies to the DescriptorSet. Otherwise, only + // the .proto files listed on the command-line are added. + bool imports_in_descriptor_set_; + // Was the --disallow_services flag used? bool disallow_services_; diff --git a/src/google/protobuf/compiler/command_line_interface_unittest.cc b/src/google/protobuf/compiler/command_line_interface_unittest.cc index 09644466..d67cbe0d 100644 --- a/src/google/protobuf/compiler/command_line_interface_unittest.cc +++ b/src/google/protobuf/compiler/command_line_interface_unittest.cc @@ -18,13 +18,23 @@ // Based on original Protocol Buffers design by // Sanjay Ghemawat, Jeff Dean, and others. +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#ifdef _MSC_VER +#include <io.h> +#else +#include <unistd.h> +#endif #include <vector> +#include <google/protobuf/descriptor.pb.h> #include <google/protobuf/descriptor.h> #include <google/protobuf/io/zero_copy_stream.h> #include <google/protobuf/compiler/command_line_interface.h> #include <google/protobuf/compiler/code_generator.h> #include <google/protobuf/io/printer.h> +#include <google/protobuf/unittest.pb.h> #include <google/protobuf/testing/file.h> #include <google/protobuf/stubs/strutil.h> @@ -35,6 +45,15 @@ namespace google { namespace protobuf { namespace compiler { +#if defined(_WIN32) +#ifndef STDIN_FILENO +#define STDIN_FILENO 0 +#endif +#ifndef STDOUT_FILENO +#define STDOUT_FILENO 1 +#endif +#endif + namespace { class CommandLineInterfaceTest : public testing::Test { @@ -110,6 +129,9 @@ class CommandLineInterfaceTest : public testing::Test { const string& message_name, const string& output_file); + void ReadDescriptorSet(const string& filename, + FileDescriptorSet* descriptor_set); + private: // The object we are testing. CommandLineInterface cli_; @@ -333,6 +355,18 @@ void CommandLineInterfaceTest::ExpectGenerated( << "Output file did not have expected contents: " + output_file; } +void CommandLineInterfaceTest::ReadDescriptorSet( + const string& filename, FileDescriptorSet* descriptor_set) { + string path = temp_directory_ + "/" + filename; + string file_contents; + if (!File::ReadFileToString(path, &file_contents)) { + FAIL() << "File not found: " << path; + } + if (!descriptor_set->ParseFromString(file_contents)) { + FAIL() << "Could not parse file contents: " << path; + } +} + // =================================================================== CommandLineInterfaceTest::MockCodeGenerator::MockCodeGenerator( @@ -665,6 +699,57 @@ TEST_F(CommandLineInterfaceTest, CwdRelativeInputs) { ExpectGenerated("test_generator", "", "foo.proto", "Foo", "output.test"); } +TEST_F(CommandLineInterfaceTest, WriteDescriptorSet) { + CreateTempFile("foo.proto", + "syntax = \"proto2\";\n" + "message Foo {}\n"); + CreateTempFile("bar.proto", + "syntax = \"proto2\";\n" + "import \"foo.proto\";\n" + "message Bar {\n" + " optional Foo foo = 1;\n" + "}\n"); + + Run("protocol_compiler --descriptor_set_out=$tmpdir/descriptor_set " + "--proto_path=$tmpdir bar.proto"); + + ExpectNoErrors(); + + FileDescriptorSet descriptor_set; + ReadDescriptorSet("descriptor_set", &descriptor_set); + if (HasFatalFailure()) return; + ASSERT_EQ(1, descriptor_set.file_size()); + EXPECT_EQ("bar.proto", descriptor_set.file(0).name()); +} + +TEST_F(CommandLineInterfaceTest, WriteTransitiveDescriptorSet) { + CreateTempFile("foo.proto", + "syntax = \"proto2\";\n" + "message Foo {}\n"); + CreateTempFile("bar.proto", + "syntax = \"proto2\";\n" + "import \"foo.proto\";\n" + "message Bar {\n" + " optional Foo foo = 1;\n" + "}\n"); + + Run("protocol_compiler --descriptor_set_out=$tmpdir/descriptor_set " + "--include_imports --proto_path=$tmpdir bar.proto"); + + ExpectNoErrors(); + + FileDescriptorSet descriptor_set; + ReadDescriptorSet("descriptor_set", &descriptor_set); + if (HasFatalFailure()) return; + ASSERT_EQ(2, descriptor_set.file_size()); + if (descriptor_set.file(0).name() == "bar.proto") { + swap(descriptor_set.mutable_file()->mutable_data()[0], + descriptor_set.mutable_file()->mutable_data()[1]); + } + EXPECT_EQ("foo.proto", descriptor_set.file(0).name()); + EXPECT_EQ("bar.proto", descriptor_set.file(1).name()); +} + // ------------------------------------------------------------------- TEST_F(CommandLineInterfaceTest, ParseErrors) { @@ -954,14 +1039,14 @@ TEST_F(CommandLineInterfaceTest, HelpText) { TEST_F(CommandLineInterfaceTest, ParseSingleCharacterFlag) { // Test that a single-character flag works. - RegisterGenerator("test_generator", "-o", + RegisterGenerator("test_generator", "-t", "output.test", "Test output."); CreateTempFile("foo.proto", "syntax = \"proto2\";\n" "message Foo {}\n"); - Run("protocol_compiler -o$tmpdir " + Run("protocol_compiler -t$tmpdir " "--proto_path=$tmpdir foo.proto"); ExpectNoErrors(); @@ -989,14 +1074,14 @@ TEST_F(CommandLineInterfaceTest, ParseSingleCharacterSpaceDelimitedValue) { // Test that separating the flag value with a space works for // single-character flags. - RegisterGenerator("test_generator", "-o", + RegisterGenerator("test_generator", "-t", "output.test", "Test output."); CreateTempFile("foo.proto", "syntax = \"proto2\";\n" "message Foo {}\n"); - Run("protocol_compiler -o $tmpdir " + Run("protocol_compiler -t $tmpdir " "--proto_path=$tmpdir foo.proto"); ExpectNoErrors(); @@ -1026,6 +1111,166 @@ TEST_F(CommandLineInterfaceTest, MissingValueAtEndError) { ExpectErrorText("Missing value for flag: --test_out\n"); } +// =================================================================== + +// Test for --encode and --decode. Note that it would be easier to do this +// test as a shell script, but we'd like to be able to run the test on +// platforms that don't have a Bourne-compatible shell available (especially +// Windows/MSVC). +class EncodeDecodeTest : public testing::Test { + protected: + virtual void SetUp() { + duped_stdin_ = dup(STDIN_FILENO); + } + + virtual void TearDown() { + dup2(duped_stdin_, STDIN_FILENO); + close(duped_stdin_); + } + + void RedirectStdinFromText(const string& input) { + string filename = TestTempDir() + "/test_stdin"; + File::WriteStringToFileOrDie(input, filename); + GOOGLE_CHECK(RedirectStdinFromFile(filename)); + } + + bool RedirectStdinFromFile(const string& filename) { + int fd = open(filename.c_str(), O_RDONLY); + if (fd < 0) return false; + dup2(fd, STDIN_FILENO); + close(fd); + return true; + } + + // Remove '\r' characters from text. + string StripCR(const string& text) { + string result; + + for (int i = 0; i < text.size(); i++) { + if (text[i] != '\r') { + result.push_back(text[i]); + } + } + + return result; + } + + enum Type { TEXT, BINARY }; + enum ReturnCode { SUCCESS, ERROR }; + + bool Run(const string& command) { + vector<string> args; + args.push_back("protoc"); + SplitStringUsing(command, " ", &args); + args.push_back("--proto_path=" + TestSourceDir()); + + scoped_array<const char*> argv(new const char*[args.size()]); + for (int i = 0; i < args.size(); i++) { + argv[i] = args[i].c_str(); + } + + CommandLineInterface cli; + cli.SetInputsAreProtoPathRelative(true); + + CaptureTestStdout(); + CaptureTestStderr(); + + int result = cli.Run(args.size(), argv.get()); + + captured_stdout_ = GetCapturedTestStdout(); + captured_stderr_ = GetCapturedTestStderr(); + + return result == 0; + } + + void ExpectStdoutMatchesBinaryFile(const string& filename) { + string expected_output; + ASSERT_TRUE(File::ReadFileToString(filename, &expected_output)); + + // Don't use EXPECT_EQ because we don't want to print raw binary data to + // stdout on failure. + EXPECT_TRUE(captured_stdout_ == expected_output); + } + + void ExpectStdoutMatchesTextFile(const string& filename) { + string expected_output; + ASSERT_TRUE(File::ReadFileToString(filename, &expected_output)); + + ExpectStdoutMatchesText(expected_output); + } + + void ExpectStdoutMatchesText(const string& expected_text) { + EXPECT_EQ(StripCR(expected_text), StripCR(captured_stdout_)); + } + + void ExpectStderrMatchesText(const string& expected_text) { + EXPECT_EQ(StripCR(expected_text), StripCR(captured_stderr_)); + } + + private: + int duped_stdin_; + string captured_stdout_; + string captured_stderr_; +}; + +TEST_F(EncodeDecodeTest, Encode) { + RedirectStdinFromFile(TestSourceDir() + + "/google/protobuf/testdata/text_format_unittest_data.txt"); + EXPECT_TRUE(Run("google/protobuf/unittest.proto " + "--encode=protobuf_unittest.TestAllTypes")); + ExpectStdoutMatchesBinaryFile(TestSourceDir() + + "/google/protobuf/testdata/golden_message"); + ExpectStderrMatchesText(""); +} + +TEST_F(EncodeDecodeTest, Decode) { + RedirectStdinFromFile(TestSourceDir() + + "/google/protobuf/testdata/golden_message"); + EXPECT_TRUE(Run("google/protobuf/unittest.proto " + "--decode=protobuf_unittest.TestAllTypes")); + ExpectStdoutMatchesTextFile(TestSourceDir() + + "/google/protobuf/testdata/text_format_unittest_data.txt"); + ExpectStderrMatchesText(""); +} + +TEST_F(EncodeDecodeTest, Partial) { + RedirectStdinFromText(""); + EXPECT_TRUE(Run("google/protobuf/unittest.proto " + "--encode=protobuf_unittest.TestRequired")); + ExpectStdoutMatchesText(""); + ExpectStderrMatchesText( + "warning: Input message is missing required fields: a, b, c\n"); +} + +TEST_F(EncodeDecodeTest, DecodeRaw) { + protobuf_unittest::TestAllTypes message; + message.set_optional_int32(123); + message.set_optional_string("foo"); + string data; + message.SerializeToString(&data); + + RedirectStdinFromText(data); + EXPECT_TRUE(Run("--decode_raw")); + ExpectStdoutMatchesText("1: 123\n" + "14: \"foo\"\n"); + ExpectStderrMatchesText(""); +} + +TEST_F(EncodeDecodeTest, UnknownType) { + EXPECT_FALSE(Run("google/protobuf/unittest.proto " + "--encode=NoSuchType")); + ExpectStdoutMatchesText(""); + ExpectStderrMatchesText("Type not defined: NoSuchType\n"); +} + +TEST_F(EncodeDecodeTest, ProtoParseError) { + EXPECT_FALSE(Run("google/protobuf/no_such_file.proto " + "--encode=NoSuchType")); + ExpectStdoutMatchesText(""); + ExpectStderrMatchesText( + "google/protobuf/no_such_file.proto: File not found.\n"); +} + } // anonymous namespace } // namespace compiler diff --git a/src/google/protobuf/compiler/cpp/cpp_file.cc b/src/google/protobuf/compiler/cpp/cpp_file.cc index aea3a4b2..f88d63fc 100644 --- a/src/google/protobuf/compiler/cpp/cpp_file.cc +++ b/src/google/protobuf/compiler/cpp/cpp_file.cc @@ -128,7 +128,14 @@ void FileGenerator::GenerateHeader(io::Printer* printer) { // Open namespace. GenerateNamespaceOpeners(printer); - printer->Print("\n"); + // Forward-declare the BuildDescriptors function, so that we can declare it + // to be a friend of each class. + printer->Print( + "\n" + "// Internal implementation detail -- do not call this.\n" + "void $builddescriptorsname$();\n" + "\n", + "builddescriptorsname", GlobalBuildDescriptorsName(file_->name())); // Generate forward declarations of classes. for (int i = 0; i < file_->message_type_count(); i++) { @@ -302,6 +309,9 @@ void FileGenerator::GenerateBuildDescriptors(io::Printer* printer) { // time, because every message has a statically-initialized default instance, // and the constructor for a message class accesses its descriptor. See the // constructor and the descriptor() method of message classes. + // + // We also construct the reflection object for each class inside + // BuildDescriptors(). printer->Print( "\n" "void $builddescriptorsname$() {\n" diff --git a/src/google/protobuf/compiler/cpp/cpp_helpers.cc b/src/google/protobuf/compiler/cpp/cpp_helpers.cc index 21de816c..6a49f815 100644 --- a/src/google/protobuf/compiler/cpp/cpp_helpers.cc +++ b/src/google/protobuf/compiler/cpp/cpp_helpers.cc @@ -188,7 +188,7 @@ string FilenameIdentifier(const string& filename) { // Return the name of the BuildDescriptors() function for a given file. string GlobalBuildDescriptorsName(const string& filename) { - return "proto_BuildDescriptors_" + FilenameIdentifier(filename); + return "protobuf_BuildDesc_" + FilenameIdentifier(filename); } } // namespace cpp diff --git a/src/google/protobuf/compiler/cpp/cpp_message.cc b/src/google/protobuf/compiler/cpp/cpp_message.cc index 002b0ad2..afd99314 100644 --- a/src/google/protobuf/compiler/cpp/cpp_message.cc +++ b/src/google/protobuf/compiler/cpp/cpp_message.cc @@ -374,6 +374,8 @@ GenerateClassDefinition(io::Printer* printer) { } else { vars["dllexport"] = dllexport_decl_ + " "; } + vars["builddescriptorsname"] = + GlobalBuildDescriptorsName(descriptor_->file()->name()); printer->Print(vars, "class $dllexport$$classname$ : public ::google::protobuf::Message {\n" @@ -396,11 +398,11 @@ GenerateClassDefinition(io::Printer* printer) { "}\n" "\n" "inline const ::google::protobuf::UnknownFieldSet& unknown_fields() const {\n" - " return _reflection_.unknown_fields();\n" + " return _unknown_fields_;\n" "}\n" "\n" "inline ::google::protobuf::UnknownFieldSet* mutable_unknown_fields() {\n" - " return _reflection_.mutable_unknown_fields();\n" + " return &_unknown_fields_;\n" "}\n" "\n" "static const ::google::protobuf::Descriptor* descriptor();\n" @@ -432,8 +434,7 @@ GenerateClassDefinition(io::Printer* printer) { "public:\n" "\n" "const ::google::protobuf::Descriptor* GetDescriptor() const;\n" - "const ::google::protobuf::Message::Reflection* GetReflection() const;\n" - "::google::protobuf::Message::Reflection* GetReflection();\n" + "const ::google::protobuf::Reflection* GetReflection() const;\n" "\n" "// nested types ----------------------------------------------------\n" "\n"); @@ -481,7 +482,7 @@ GenerateClassDefinition(io::Printer* printer) { // TODO(kenton): Make _cached_size_ an atomic<int> when C++ supports it. printer->Print( - "::google::protobuf::internal::GeneratedMessageReflection _reflection_;\n" + "::google::protobuf::UnknownFieldSet _unknown_fields_;\n" "mutable int _cached_size_;\n" "\n"); for (int i = 0; i < descriptor_->field_count(); i++) { @@ -491,7 +492,7 @@ GenerateClassDefinition(io::Printer* printer) { // Generate offsets and _has_bits_ boilerplate. printer->Print(vars, - "\n" + "friend void $builddescriptorsname$();\n" "static const $classname$ default_instance_;\n"); if (descriptor_->field_count() > 0) { @@ -540,8 +541,11 @@ GenerateInlineMethods(io::Printer* printer) { void MessageGenerator:: GenerateDescriptorDeclarations(io::Printer* printer) { - printer->Print("const ::google::protobuf::Descriptor* $name$_descriptor_ = NULL;\n", - "name", classname_); + printer->Print( + "const ::google::protobuf::Descriptor* $name$_descriptor_ = NULL;\n" + "const ::google::protobuf::internal::GeneratedMessageReflection*\n" + " $name$_reflection_ = NULL;\n", + "name", classname_); for (int i = 0; i < descriptor_->nested_type_count(); i++) { nested_generators_[i]->GenerateDescriptorDeclarations(printer); @@ -562,6 +566,7 @@ GenerateDescriptorInitializer(io::Printer* printer, int index) { vars["classname"] = classname_; vars["index"] = SimpleItoa(index); + // Obtain the descriptor from the parent's descriptor. if (descriptor_->containing_type() == NULL) { printer->Print(vars, "$classname$_descriptor_ = file->message_type($index$);\n"); @@ -572,6 +577,29 @@ GenerateDescriptorInitializer(io::Printer* printer, int index) { "$parent$_descriptor_->nested_type($index$);\n"); } + // Construct the reflection object. + printer->Print(vars, + "$classname$_reflection_ =\n" + " new ::google::protobuf::internal::GeneratedMessageReflection(\n" + " $classname$_descriptor_,\n" + " &$classname$::default_instance(),\n" + " $classname$::_offsets_,\n" + " GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET($classname$, _has_bits_[0]),\n" + " GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(" + "$classname$, _unknown_fields_),\n"); + if (descriptor_->extension_range_count() > 0) { + printer->Print(vars, + " GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(" + "$classname$, _extensions_),\n"); + } else { + // No extensions. + printer->Print(vars, + " -1,\n"); + } + printer->Print(vars, + " ::google::protobuf::DescriptorPool::generated_pool());\n"); + + // Handle nested types. for (int i = 0; i < descriptor_->nested_type_count(); i++) { nested_generators_[i]->GenerateDescriptorInitializer(printer, i); } @@ -650,15 +678,13 @@ GenerateClassMethods(io::Printer* printer) { " return descriptor();\n" "}\n" "\n" - "const ::google::protobuf::Message::Reflection*\n" - "$classname$::GetReflection() const {\n" - " return &_reflection_;\n" - "}\n" - "\n" - "::google::protobuf::Message::Reflection* $classname$::GetReflection() {\n" - " return &_reflection_;\n" + "const ::google::protobuf::Reflection* $classname$::GetReflection() const {\n" + " if ($classname$_reflection_ == NULL) $builddescriptorsname$();\n" + " return $classname$_reflection_;\n" "}\n", - "classname", classname_); + "classname", classname_, + "builddescriptorsname", + GlobalBuildDescriptorsName(descriptor_->file()->name())); } void MessageGenerator:: @@ -686,20 +712,16 @@ GenerateInitializerList(io::Printer* printer) { printer->Indent(); printer->Indent(); - bool has_extensions = descriptor_->extension_range_count() > 0; - if (has_extensions) { + if (descriptor_->extension_range_count() > 0) { printer->Print( - "_extensions_(descriptor(),\n" + "_extensions_(&$classname$_descriptor_,\n" " ::google::protobuf::DescriptorPool::generated_pool(),\n" - " ::google::protobuf::MessageFactory::generated_factory()),\n"); + " ::google::protobuf::MessageFactory::generated_factory()),\n", + "classname", classname_); } printer->Print( - "_reflection_(descriptor(),\n" - " this, &default_instance_,\n" - " _offsets_, _has_bits_, $extensions$),\n" - "_cached_size_(0)", - "extensions", has_extensions ? "&_extensions_" : "NULL"); + "_cached_size_(0)"); // Write the initializers for each field. for (int i = 0; i < descriptor_->field_count(); i++) { @@ -904,8 +926,7 @@ GenerateMergeFrom(io::Printer* printer) { " ::google::protobuf::internal::dynamic_cast_if_available<const $classname$*>(\n" " &from);\n" "if (source == NULL) {\n" - " ::google::protobuf::internal::ReflectionOps::Merge(\n" - " descriptor(), *from.GetReflection(), &_reflection_);\n" + " ::google::protobuf::internal::ReflectionOps::Merge(from, this);\n" "} else {\n" " MergeFrom(*source);\n" "}\n", @@ -1028,7 +1049,7 @@ GenerateMergeFromCodedStream(io::Printer* printer) { "bool $classname$::MergePartialFromCodedStream(\n" " ::google::protobuf::io::CodedInputStream* input) {\n" " return ::google::protobuf::internal::WireFormat::ParseAndMergePartial(\n" - " descriptor(), input, &_reflection_);\n" + " input, this);\n" "}\n", "classname", classname_); return; @@ -1157,7 +1178,7 @@ GenerateMergeFromCodedStream(io::Printer* printer) { } } printer->Print(") {\n" - " DO_(_extensions_.ParseField(tag, input, &_reflection_));\n" + " DO_(_extensions_.ParseField(tag, input, this));\n" " continue;\n" "}\n"); } @@ -1214,7 +1235,7 @@ void MessageGenerator::GenerateSerializeOneExtensionRange( printer->Print(vars, "// Extension range [$start$, $end$)\n" "DO_(_extensions_.SerializeWithCachedSizes(\n" - " $start$, $end$, &_reflection_, output));\n\n"); + " $start$, $end$, *this, output));\n\n"); } void MessageGenerator:: @@ -1341,7 +1362,7 @@ GenerateByteSize(io::Printer* printer) { if (descriptor_->extension_range_count() > 0) { printer->Print( - "total_size += _extensions_.ByteSize(&_reflection_);\n" + "total_size += _extensions_.ByteSize(*this);\n" "\n"); } diff --git a/src/google/protobuf/compiler/cpp/cpp_unittest.cc b/src/google/protobuf/compiler/cpp/cpp_unittest.cc index 561a5ad1..010843cf 100644 --- a/src/google/protobuf/compiler/cpp/cpp_unittest.cc +++ b/src/google/protobuf/compiler/cpp/cpp_unittest.cc @@ -268,7 +268,7 @@ TEST(GeneratedMessageTest, DynamicMessageCopyFrom) { TestUtil::ReflectionTester reflection_tester( unittest::TestAllTypes::descriptor()); - reflection_tester.SetAllFieldsViaReflection(message1->GetReflection()); + reflection_tester.SetAllFieldsViaReflection(message1.get()); message2.CopyFrom(*message1); diff --git a/src/google/protobuf/compiler/parser.cc b/src/google/protobuf/compiler/parser.cc index 3b73530b..7a148c5a 100644 --- a/src/google/protobuf/compiler/parser.cc +++ b/src/google/protobuf/compiler/parser.cc @@ -604,7 +604,7 @@ bool Parser::ParseDefaultAssignment(FieldDescriptorProto* field) { } bool Parser::ParseOptionAssignment(Message* options) { - Message::Reflection* reflection = options->GetReflection(); + const Reflection* reflection = options->GetReflection(); const Descriptor* descriptor = options->GetDescriptor(); // Parse name. @@ -623,7 +623,7 @@ bool Parser::ParseOptionAssignment(Message* options) { AddError(line, column, "Not implemented: repeated options."); return false; } - if (reflection->HasField(field)) { + if (reflection->HasField(*options, field)) { AddError(line, column, "Option \"" + name + "\" was already set."); return false; } @@ -638,7 +638,7 @@ bool Parser::ParseOptionAssignment(Message* options) { // This field is a message/group. The user must identify a field within // it to set. - return ParseOptionAssignment(reflection->MutableMessage(field)); + return ParseOptionAssignment(reflection->MutableMessage(options, field)); } DO(Consume("=")); @@ -651,7 +651,7 @@ bool Parser::ParseOptionAssignment(Message* options) { uint64 max_value = kint32max; if (is_negative) ++max_value; DO(ConsumeInteger64(max_value, &value, "Expected integer.")); - reflection->SetInt32(field, is_negative ? -value : value); + reflection->SetInt32(options, field, is_negative ? -value : value); break; } @@ -661,21 +661,21 @@ bool Parser::ParseOptionAssignment(Message* options) { uint64 max_value = kint64max; if (is_negative) ++max_value; DO(ConsumeInteger64(max_value, &value, "Expected integer.")); - reflection->SetInt64(field, is_negative ? -value : value); + reflection->SetInt64(options, field, is_negative ? -value : value); break; } case FieldDescriptor::CPPTYPE_UINT32: { uint64 value; DO(ConsumeInteger64(kuint32max, &value, "Expected integer.")); - reflection->SetUInt32(field, value); + reflection->SetUInt32(options, field, value); break; } case FieldDescriptor::CPPTYPE_UINT64: { uint64 value; DO(ConsumeInteger64(kuint64max, &value, "Expected integer.")); - reflection->SetUInt64(field, value); + reflection->SetUInt64(options, field, value); break; } @@ -683,7 +683,7 @@ bool Parser::ParseOptionAssignment(Message* options) { double value; bool is_negative = TryConsume("-"); DO(ConsumeNumber(&value, "Expected number.")); - reflection->SetDouble(field, is_negative ? -value : value); + reflection->SetDouble(options, field, is_negative ? -value : value); break; } @@ -691,15 +691,15 @@ bool Parser::ParseOptionAssignment(Message* options) { double value; bool is_negative = TryConsume("-"); DO(ConsumeNumber(&value, "Expected number.")); - reflection->SetFloat(field, is_negative ? -value : value); + reflection->SetFloat(options, field, is_negative ? -value : value); break; } case FieldDescriptor::CPPTYPE_BOOL: if (TryConsume("true")) { - reflection->SetBool(field, true); + reflection->SetBool(options, field, true); } else if (TryConsume("false")) { - reflection->SetBool(field, false); + reflection->SetBool(options, field, false); } else { AddError("Expected \"true\" or \"false\"."); return false; @@ -719,14 +719,14 @@ bool Parser::ParseOptionAssignment(Message* options) { "named \"" + value_name + "\"."); return false; } - reflection->SetEnum(field, value); + reflection->SetEnum(options, field, value); break; } case FieldDescriptor::CPPTYPE_STRING: { string value; DO(ConsumeString(&value, "Expected string.")); - reflection->SetString(field, value); + reflection->SetString(options, field, value); break; } |