From e8e1dab03f8cdd473a3504c4efbe6920eb2c2612 Mon Sep 17 00:00:00 2001 From: csharptest Date: Fri, 3 Sep 2010 16:36:50 -0500 Subject: Completed the following changes & testing, see todo.txt for more information 1 - Add a way to specify the output directory 2 - Added an option "file_extension" to control the suffix for cs files generated, defaults to ".cs" 3 - Added the option for "umbrella_namespace" used when nest_classes=false and having name conflicts 4 - Optionally remove dependencies to csharp options 5 - Investigate command line parsing library 6 - Investigate calling protoc directly 7 - Unable to resolve dependencies correctly 8 - Added several (20) nunits to automate the command-line invocation of each option --- src/ProtoGen/DescriptorUtil.cs | 16 ++- src/ProtoGen/Generator.cs | 70 ++++++++++-- src/ProtoGen/GeneratorOptions.cs | 190 ++++++++++++++++++++++++++++++++- src/ProtoGen/Program.cs | 9 +- src/ProtoGen/ProgramPreprocess.cs | 150 ++++++++++++++++++++++++++ src/ProtoGen/ProtoGen.csproj | 2 + src/ProtoGen/ServiceGenerator.cs | 2 +- src/ProtoGen/UmbrellaClassGenerator.cs | 17 ++- 8 files changed, 433 insertions(+), 23 deletions(-) create mode 100644 src/ProtoGen/ProgramPreprocess.cs (limited to 'src/ProtoGen') diff --git a/src/ProtoGen/DescriptorUtil.cs b/src/ProtoGen/DescriptorUtil.cs index 14d28bb8..f485db6f 100644 --- a/src/ProtoGen/DescriptorUtil.cs +++ b/src/ProtoGen/DescriptorUtil.cs @@ -46,10 +46,24 @@ namespace Google.ProtocolBuffers.ProtoGen { CSharpFileOptions options = descriptor.File.CSharpOptions; string result = options.Namespace; if (result != "") result += '.'; - result += options.UmbrellaClassname; + result += QualifiedUmbrellaClassName(options); return "global::" + result; } + /// + /// ROK 2010-09-03 + /// Evaluates the options and returns the qualified name of the umbrella class + /// relative to the descriptor type's namespace. Basically contacts the + /// UmbrellaNamespace + UmbrellaClassname fields. + /// + internal static string QualifiedUmbrellaClassName(CSharpFileOptions options) + { + string fullName = options.UmbrellaClassname; + if (!options.NestClasses && options.UmbrellaNamespace != "") + fullName = String.Format("{0}.{1}", options.UmbrellaNamespace, options.UmbrellaClassname); + return fullName; + } + internal static string GetMappedTypeName(MappedType type) { switch(type) { case MappedType.Int32: return "int"; diff --git a/src/ProtoGen/Generator.cs b/src/ProtoGen/Generator.cs index 7e48ad67..b2d4e23f 100644 --- a/src/ProtoGen/Generator.cs +++ b/src/ProtoGen/Generator.cs @@ -60,19 +60,30 @@ namespace Google.ProtocolBuffers.ProtoGen { } public void Generate() { + + List descriptorProtos = new List(); foreach (string inputFile in options.InputFiles) { - FileDescriptorSet descriptorProtos; ExtensionRegistry extensionRegistry = ExtensionRegistry.CreateInstance(); extensionRegistry.Add(CSharpOptions.CSharpFileOptions); extensionRegistry.Add(CSharpOptions.CSharpFieldOptions); using (Stream inputStream = File.OpenRead(inputFile)) { - descriptorProtos = FileDescriptorSet.ParseFrom(inputStream, extensionRegistry); + descriptorProtos.Add(FileDescriptorSet.ParseFrom(inputStream, extensionRegistry)); } - IList descriptors = ConvertDescriptors(descriptorProtos); + } + + IList descriptors = ConvertDescriptors(options.FileOptions, descriptorProtos.ToArray()); + + //ROK Combine with Options... + foreach (FileDescriptor descriptor in descriptors) + descriptor.ConfigureWithDefaultOptions(options.FileOptions); foreach (FileDescriptor descriptor in descriptors) { + //ROK 2010-09-03 Ignore google protobuf package + if(descriptor.CSharpOptions.IgnoreGoogleProtobuf && descriptor.Package == "google.protobuf") + continue; + Generate(descriptor); - } + } } @@ -90,8 +101,14 @@ namespace Google.ProtocolBuffers.ProtoGen { private string GetOutputFile(FileDescriptor descriptor) { CSharpFileOptions fileOptions = descriptor.CSharpOptions; - string filename = descriptor.CSharpOptions.UmbrellaClassname + ".cs"; - string outputDirectory = options.OutputDirectory; + + //ROK 2010-09-03 - added the ability to sepcify the extension used within the options + //string filename = descriptor.CSharpOptions.UmbrellaClassname + ".cs"; + string filename = descriptor.CSharpOptions.UmbrellaClassname + descriptor.CSharpOptions.FileExtension; + + //ROK 2010-09-03 - output directory can be specific to a descriptor file + //string outputDirectory = options.OutputDirectory; + string outputDirectory = descriptor.CSharpOptions.OutputDirectory; if (fileOptions.ExpandNamespaceDirectories) { string package = fileOptions.Namespace; if (!string.IsNullOrEmpty(package)) { @@ -99,22 +116,31 @@ namespace Google.ProtocolBuffers.ProtoGen { foreach (string bit in bits) { outputDirectory = Path.Combine(outputDirectory, bit); } - Directory.CreateDirectory(outputDirectory); } - } + } + //ROK 2010-09-03 - Always force output directory exists since they can specify this in .proto options + Directory.CreateDirectory(outputDirectory); return Path.Combine(outputDirectory, filename); } + // ROK 2010-09-03 - used by unit tests, we will continue to allow them to function as-is. + internal static IList ConvertDescriptors(FileDescriptorSet descriptorProtos) { + return ConvertDescriptors(CSharpFileOptions.DefaultInstance, descriptorProtos); + } + /// /// Resolves any dependencies and converts FileDescriptorProtos into FileDescriptors. /// The list returned is in the same order as the protos are listed in the descriptor set. /// Note: this method is internal rather than private to allow testing. /// /// Not all dependencies could be resolved. - internal static IList ConvertDescriptors(FileDescriptorSet descriptorProtos) { + private static IList ConvertDescriptors(CSharpFileOptions options, params FileDescriptorSet[] descriptorProtos) { // Simple strategy: Keep going through the list of protos to convert, only doing ones where // we've already converted all the dependencies, until we get to a stalemate - IList fileList = descriptorProtos.FileList; + List fileList = new List(); + foreach (FileDescriptorSet set in descriptorProtos) + fileList.AddRange(set.FileList); + FileDescriptor[] converted = new FileDescriptor[fileList.Count]; Dictionary convertedMap = new Dictionary(); @@ -131,10 +157,32 @@ namespace Google.ProtocolBuffers.ProtoGen { } FileDescriptorProto candidate = fileList[i]; FileDescriptor[] dependencies = new FileDescriptor[candidate.DependencyList.Count]; + + + CSharpFileOptions.Builder builder = options.ToBuilder(); + if (candidate.Options.HasExtension(DescriptorProtos.CSharpOptions.CSharpFileOptions)) { + builder.MergeFrom(candidate.Options.GetExtension(DescriptorProtos.CSharpOptions.CSharpFileOptions)); + } + CSharpFileOptions localOptions = builder.Build(); + bool foundAllDependencies = true; for (int j = 0; j < dependencies.Length; j++) { if (!convertedMap.TryGetValue(candidate.DependencyList[j], out dependencies[j])) { - foundAllDependencies = false; + + // ROK 2010-09-03 - we can auto-magically resolve these since we already have their description + // This way if the file is only referencing options it does not need to be built with the + // --include_imports definition. + if (localOptions.IgnoreGoogleProtobuf && (candidate.DependencyList[j] == "google/protobuf/csharp_options.proto")) + { + dependencies[j] = CSharpOptions.Descriptor; + continue; + } + if (localOptions.IgnoreGoogleProtobuf && (candidate.DependencyList[j] == "google/protobuf/descriptor.proto")) + { + dependencies[j] = DescriptorProtoFile.Descriptor; + continue; + } + foundAllDependencies = false; break; } } diff --git a/src/ProtoGen/GeneratorOptions.cs b/src/ProtoGen/GeneratorOptions.cs index 1f3d8f2b..53c65a40 100644 --- a/src/ProtoGen/GeneratorOptions.cs +++ b/src/ProtoGen/GeneratorOptions.cs @@ -32,8 +32,12 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endregion +using System; using System.Collections.Generic; using System.IO; +using System.Text.RegularExpressions; +using Google.ProtocolBuffers.DescriptorProtos; +using Google.ProtocolBuffers.Descriptors; namespace Google.ProtocolBuffers.ProtoGen { @@ -44,8 +48,7 @@ namespace Google.ProtocolBuffers.ProtoGen { /// the generator. /// public sealed class GeneratorOptions { - - public string OutputDirectory { get; set; } + //ROK, see below - public string OutputDirectory { get; set; } public IList InputFiles { get; set; } /// @@ -58,12 +61,15 @@ namespace Google.ProtocolBuffers.ProtoGen { public bool TryValidate(out IList reasons) { List tmpReasons = new List(); + //ROK 2010-09-03 see population of options below + ParseArguments(tmpReasons); + // Output directory validation - if (string.IsNullOrEmpty(OutputDirectory)) { + if (string.IsNullOrEmpty(FileOptions.OutputDirectory)) { tmpReasons.Add("No output directory specified"); } else { - if (!Directory.Exists(OutputDirectory)) { - tmpReasons.Add("Specified output directory (" + OutputDirectory + " doesn't exist."); + if (!Directory.Exists(FileOptions.OutputDirectory)) { + tmpReasons.Add("Specified output directory (" + FileOptions.OutputDirectory + " doesn't exist."); } } @@ -99,5 +105,179 @@ namespace Google.ProtocolBuffers.ProtoGen { throw new InvalidOptionsException(reasons); } } + + + + // ROK - added to provide defaults for any of the options + //Raw arguments + public IList Arguments { get; set; } + [Obsolete("Please use GeneratorOptions.FileOptions.OutputDirectory instead")] + public string OutputDirectory + { + get { + return FileOptions.OutputDirectory; + } + set { + CSharpFileOptions.Builder bld = FileOptions.ToBuilder(); + bld.OutputDirectory = value; + FileOptions = bld.Build(); + } + } + + private static readonly Regex ArgMatch = new Regex(@"^[-/](?[\w_]+?)[:=](?.*)$"); + CSharpFileOptions _fileOptions; + public CSharpFileOptions FileOptions + { + get { return _fileOptions ?? (_fileOptions = CSharpFileOptions.DefaultInstance); } + set { _fileOptions = value; } + } + + private void ParseArguments(IList tmpReasons) + { + bool doHelp = Arguments.Count == 0; + + //ROK Parse the raw arguments + InputFiles = new List(); + CSharpFileOptions.Builder builder = FileOptions.ToBuilder(); + Dictionary fields = + new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (FieldDescriptor fld in builder.DescriptorForType.Fields) + fields.Add(fld.Name, fld); + + foreach (string argument in Arguments) + { + if (StringComparer.OrdinalIgnoreCase.Equals("-help", argument) || + StringComparer.OrdinalIgnoreCase.Equals("/help", argument) || + StringComparer.OrdinalIgnoreCase.Equals("-?", argument) || + StringComparer.OrdinalIgnoreCase.Equals("/?", argument)) + { + doHelp = true; + break; + } + + Match m = ArgMatch.Match(argument); + if (m.Success) + { + FieldDescriptor fld; + string name = m.Groups["name"].Value; + string value = m.Groups["value"].Value; + + if (fields.TryGetValue(name, out fld)) + { + object obj; + if (TryCoerceType(value, fld, out obj, tmpReasons)) + builder[fld] = obj; + } + else if (!File.Exists(argument)) + { + doHelp = true; + tmpReasons.Add("Unknown argument '" + name + "'."); + } + else + InputFiles.Add(argument); + } + else + InputFiles.Add(argument); + } + + if (doHelp || InputFiles.Count == 0) + { + tmpReasons.Add("Arguments:"); + foreach (KeyValuePair field in fields) + { + tmpReasons.Add(String.Format("-{0}=[{1}]", field.Key, field.Value.FieldType)); + } + tmpReasons.Add("followed by one or more file paths."); + } + else + FileOptions = builder.Build(); + } + + private static bool TryCoerceType(string text, FieldDescriptor field, out object value, IList tmpReasons) + { + value = null; + + switch (field.FieldType) + { + case FieldType.Int32: + case FieldType.SInt32: + case FieldType.SFixed32: + value = Int32.Parse(text); + break; + + case FieldType.Int64: + case FieldType.SInt64: + case FieldType.SFixed64: + value = Int64.Parse(text); + break; + + case FieldType.UInt32: + case FieldType.Fixed32: + value = UInt32.Parse(text); + break; + + case FieldType.UInt64: + case FieldType.Fixed64: + value = UInt64.Parse(text); + break; + + case FieldType.Float: + value = float.Parse(text); + break; + + case FieldType.Double: + value = Double.Parse(text); + break; + + case FieldType.Bool: + value = Boolean.Parse(text); + break; + + case FieldType.String: + value = text; + break; + + case FieldType.Enum: + { + + EnumDescriptor enumType = field.EnumType; + + int number; + if (int.TryParse(text, out number)) + { + value = enumType.FindValueByNumber(number); + if (value == null) + { + tmpReasons.Add( + "Enum type \"" + enumType.FullName + + "\" has no value with number " + number + "."); + return false; + } + } + else + { + value = enumType.FindValueByName(text); + if (value == null) + { + tmpReasons.Add( + "Enum type \"" + enumType.FullName + + "\" has no value named \"" + text + "\"."); + return false; + } + } + + break; + } + + case FieldType.Bytes: + case FieldType.Message: + case FieldType.Group: + tmpReasons.Add("Unhandled field type " + field.FieldType.ToString() + "."); + return false; + } + + return true; + } + } } diff --git a/src/ProtoGen/Program.cs b/src/ProtoGen/Program.cs index f9f80f25..a9bc47ce 100644 --- a/src/ProtoGen/Program.cs +++ b/src/ProtoGen/Program.cs @@ -41,7 +41,7 @@ namespace Google.ProtocolBuffers.ProtoGen { /// Entry point for the Protocol Buffers generator. /// class Program { - static int Main(string[] args) { + internal static int Main(string[] args) { try { // Hack to make sure everything's initialized DescriptorProtoFile.Descriptor.ToString(); @@ -71,8 +71,11 @@ namespace Google.ProtocolBuffers.ProtoGen { //string baseDir = "c:\\Users\\Jon\\Documents\\Visual Studio 2008\\Projects\\ProtocolBuffers"; //options.OutputDirectory = baseDir + "\\tmp"; //options.InputFiles = new[] { baseDir + "\\protos\\nwind-solo.protobin" }; - options.OutputDirectory = "."; - options.InputFiles = args; + + //ROK 2010-09-03 - fixes to allow parsing these options... + //options.OutputDirectory = "."; + //options.InputFiles = args; + options.Arguments = args; return options; } } diff --git a/src/ProtoGen/ProgramPreprocess.cs b/src/ProtoGen/ProgramPreprocess.cs new file mode 100644 index 00000000..2081b016 --- /dev/null +++ b/src/ProtoGen/ProgramPreprocess.cs @@ -0,0 +1,150 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; + +namespace Google.ProtocolBuffers.ProtoGen +{ + /// + /// Preprocesses any input files with an extension of '.proto' by running protoc.exe. If arguments + /// are supplied with '--' prefix they are provided to protoc.exe, otherwise they are assumed to + /// be used for ProtoGen.exe which is run on the resulting output proto buffer. If the option + /// --descriptor_set_out= is specified the proto buffer file is kept, otherwise it will be removed + /// after code generation. + /// + internal class ProgramPreprocess + { + static int Main(string[] args) + { + try + { + return Environment.ExitCode = Run(args); + } + catch (Exception ex) + { + Console.Error.WriteLine(ex); + return Environment.ExitCode = 2; + } + } + + internal static int Run(params string[] args) + { + bool deleteFile = false; + string tempFile = null; + int result = 1; + bool doHelp = args.Length == 0; + try + { + List protocArgs = new List(); + List protoGenArgs = new List(); + + foreach (string arg in args) + { + doHelp |= StringComparer.OrdinalIgnoreCase.Equals(arg, "/?"); + doHelp |= StringComparer.OrdinalIgnoreCase.Equals(arg, "/help"); + doHelp |= StringComparer.OrdinalIgnoreCase.Equals(arg, "-?"); + doHelp |= StringComparer.OrdinalIgnoreCase.Equals(arg, "-help"); + + if(arg.StartsWith("--descriptor_set_out=")) + { + tempFile = arg.Substring("--descriptor_set_out=".Length); + protoGenArgs.Add(tempFile); + } + } + + if (doHelp) + { + Console.WriteLine(); + Console.WriteLine("PROTOC.exe: Use any of the following options that begin with '--':"); + Console.WriteLine(); + try { RunProtoc("--help"); } + catch (Exception ex) { Console.Error.WriteLine(ex.Message); } + Console.WriteLine(); + Console.WriteLine(); + Console.WriteLine("PRTOGEN.exe: The following options are used to specify defaults for code generation."); + Console.WriteLine(); + Program.Main(new string[0]); + return 0; + } + + foreach (string arg in args) + { + if (arg.StartsWith("--")) + protocArgs.Add(arg); + else if (File.Exists(arg) && StringComparer.OrdinalIgnoreCase.Equals(".proto", Path.GetExtension(arg))) + { + if (tempFile == null) + { + deleteFile = true; + tempFile = Path.GetTempFileName(); + protocArgs.Add(String.Format("--descriptor_set_out={0}", tempFile)); + protoGenArgs.Add(tempFile); + } + protocArgs.Add(arg); + } + else + protoGenArgs.Add(arg); + } + + if (tempFile != null) + { + result = RunProtoc(protocArgs.ToArray()); + if (result != 0) + return result; + } + + result = Program.Main(protoGenArgs.ToArray()); + } + finally + { + if (deleteFile && tempFile != null && File.Exists(tempFile)) + File.Delete(tempFile); + } + return result; + } + + private static int RunProtoc(params string[] args) + { + const string protoc = "protoc.exe"; + string exePath = protoc; + + //why oh why is this not in System.IO.Path or Environment... + List searchPath = new List(); + searchPath.Add(Environment.CurrentDirectory); + searchPath.Add(AppDomain.CurrentDomain.BaseDirectory); + searchPath.AddRange((Environment.GetEnvironmentVariable("PATH") ?? String.Empty).Split(Path.PathSeparator)); + + foreach (string path in searchPath) + if (File.Exists(exePath = Path.Combine(path, protoc))) + break; + + if (!File.Exists(exePath)) + throw new FileNotFoundException("Unable to locate " + protoc + " make sure it is in the PATH, cwd, or exe dir."); + + for (int i = 0; i < args.Length; i++) + if (args[i].IndexOf(' ') > 0 && args[i][0] != '"') + args[i] = '"' + args[i] + '"'; + + ProcessStartInfo psi = new ProcessStartInfo(exePath); + psi.Arguments = String.Join(" ", args); + psi.RedirectStandardError = true; + psi.RedirectStandardInput = false; + psi.RedirectStandardOutput = true; + psi.ErrorDialog = false; + psi.CreateNoWindow = true; + psi.UseShellExecute = false; + psi.WorkingDirectory = Environment.CurrentDirectory; + + Process process = Process.Start(psi); + if (process == null) return 1; + + process.WaitForExit(); + + string tmp = process.StandardOutput.ReadToEnd(); + if(tmp.Trim().Length > 0) Console.Out.WriteLine(tmp); + tmp = process.StandardError.ReadToEnd(); + if (tmp.Trim().Length > 0) Console.Error.WriteLine(tmp); + return process.ExitCode; + } + } +} diff --git a/src/ProtoGen/ProtoGen.csproj b/src/ProtoGen/ProtoGen.csproj index 056b2c36..03b81ba5 100644 --- a/src/ProtoGen/ProtoGen.csproj +++ b/src/ProtoGen/ProtoGen.csproj @@ -16,6 +16,7 @@ true Properties\Google.ProtocolBuffers.ProtoGen.snk + Google.ProtocolBuffers.ProtoGen.ProgramPreprocess true @@ -58,6 +59,7 @@ + diff --git a/src/ProtoGen/ServiceGenerator.cs b/src/ProtoGen/ServiceGenerator.cs index a7a4471e..83d8a6a3 100644 --- a/src/ProtoGen/ServiceGenerator.cs +++ b/src/ProtoGen/ServiceGenerator.cs @@ -61,7 +61,7 @@ namespace Google.ProtocolBuffers.ProtoGen { writer.WriteLine(); writer.WriteLine("{0} static pbd::ServiceDescriptor Descriptor {{", ClassAccessLevel); writer.WriteLine(" get {{ return {0}.Descriptor.Services[{1}]; }}", - Descriptor.File.CSharpOptions.UmbrellaClassname, Descriptor.Index); + DescriptorUtil.QualifiedUmbrellaClassName(Descriptor.File.CSharpOptions), Descriptor.Index); writer.WriteLine("}"); writer.WriteLine("{0} pbd::ServiceDescriptor DescriptorForType {{", ClassAccessLevel); writer.WriteLine(" get { return Descriptor; }"); diff --git a/src/ProtoGen/UmbrellaClassGenerator.cs b/src/ProtoGen/UmbrellaClassGenerator.cs index 9dcbcab9..d596f1a1 100644 --- a/src/ProtoGen/UmbrellaClassGenerator.cs +++ b/src/ProtoGen/UmbrellaClassGenerator.cs @@ -92,7 +92,13 @@ namespace Google.ProtocolBuffers.ProtoGen { if (!Descriptor.CSharpOptions.NestClasses) { writer.Outdent(); writer.WriteLine("}"); - } + + //ROK 2010-09-03 - close the namespace around the umbrella class if defined + if (!Descriptor.CSharpOptions.NestClasses && Descriptor.CSharpOptions.UmbrellaNamespace != "") { + writer.Outdent(); + writer.WriteLine("}"); + } + } WriteChildren(writer, "Enums", Descriptor.EnumTypes); WriteChildren(writer, "Messages", Descriptor.MessageTypes); WriteChildren(writer, "Services", Descriptor.Services); @@ -115,7 +121,14 @@ namespace Google.ProtocolBuffers.ProtoGen { writer.WriteLine("namespace {0} {{", Descriptor.CSharpOptions.Namespace); writer.Indent(); writer.WriteLine(); - } + } + //ROK 2010-09-03 - add the namespace around the umbrella class if defined + if(!Descriptor.CSharpOptions.NestClasses && Descriptor.CSharpOptions.UmbrellaNamespace != "") { + writer.WriteLine("namespace {0} {{", Descriptor.CSharpOptions.UmbrellaNamespace); + writer.Indent(); + writer.WriteLine(); + } + if (Descriptor.CSharpOptions.CodeContracts) { writer.WriteLine("[global::System.Diagnostics.Contracts.ContractVerificationAttribute(false)]"); } -- cgit v1.2.3