From 993fb7013e0139ec7b16abcada7995f3a642e331 Mon Sep 17 00:00:00 2001 From: Jisi Liu Date: Mon, 19 Oct 2015 17:19:49 -0700 Subject: Python bazel support. --- BUILD | 149 +++++++++++++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 120 insertions(+), 29 deletions(-) (limited to 'BUILD') diff --git a/BUILD b/BUILD index 69ea6cf3..465628fa 100644 --- a/BUILD +++ b/BUILD @@ -18,7 +18,8 @@ COPTS = [ # Bazel should provide portable link_opts for pthread. LINK_OPTS = ["-lpthread"] -load("protobuf", "cc_proto_library") +load("protobuf", "cc_proto_library", "py_proto_library", "copied_srcs", + "internal_protobuf_py_tests") cc_library( name = "protobuf_lite", @@ -126,7 +127,7 @@ objc_library( visibility = ["//visibility:public"], ) -WELL_KNOWN_PROTOS = [ +RELATIVE_WELL_KNOWN_PROTOS = [ # AUTOGEN(well_known_protos) "google/protobuf/any.proto", "google/protobuf/api.proto", @@ -142,12 +143,14 @@ WELL_KNOWN_PROTOS = [ "google/protobuf/wrappers.proto", ] +WELL_KNOWN_PROTOS = ["src/" + s for s in RELATIVE_WELL_KNOWN_PROTOS] + cc_proto_library( name = "cc_wkt_protos", - srcs = ["src/" + s for s in WELL_KNOWN_PROTOS], - internal_bootstrap_hack = 1, + srcs = WELL_KNOWN_PROTOS, include = "src", cc_libs = [":protobuf"], + internal_bootstrap_hack = 1, ) ################################################################################ @@ -263,32 +266,11 @@ cc_binary( deps = [":protoc_lib"], ) -################################################################################ -# Java support -################################################################################ -genrule( - name = "generate_java_descriptor_proto", - srcs = ["src/google/protobuf/descriptor.proto"], - outs = ["com/google/protobuf/DescriptorProtos.java"], - cmd = "$(location :protoc) --java_out=$(@D)/../../.. $<", - tools = [":protoc"], -) - -java_library( - name = "java_proto", - srcs = glob([ - "java/src/main/java/com/google/protobuf/*.java", - ]) + [ - ":generate_java_descriptor_proto", - ], - visibility = ["//visibility:public"], -) - ################################################################################ # Tests ################################################################################ -LITE_TEST_PROTOS = [ +RELATIVE_LITE_TEST_PROTOS = [ # AUTOGEN(lite_test_protos) "google/protobuf/map_lite_unittest.proto", "google/protobuf/unittest_import_lite.proto", @@ -297,7 +279,9 @@ LITE_TEST_PROTOS = [ "google/protobuf/unittest_no_arena_lite.proto", ] -TEST_PROTOS = [ +LITE_TEST_PROTOS = ["src/" + s for s in RELATIVE_LITE_TEST_PROTOS] + +RELATIVE_TEST_PROTOS = [ # AUTOGEN(test_protos) "google/protobuf/any_test.proto", "google/protobuf/compiler/cpp/cpp_test_bad_identifiers.proto", @@ -337,9 +321,11 @@ TEST_PROTOS = [ "google/protobuf/util/json_format_proto3.proto", ] +TEST_PROTOS = ["src/" + s for s in RELATIVE_TEST_PROTOS] + cc_proto_library( name = "cc_test_protos", - srcs = ["src/" + s for s in (LITE_TEST_PROTOS + TEST_PROTOS)], + srcs = LITE_TEST_PROTOS + TEST_PROTOS, include = "src", deps = [":cc_wkt_protos"], ) @@ -445,9 +431,114 @@ cc_test( ], linkopts = LINK_OPTS, deps = [ + ":cc_test_protos", ":protobuf", ":protoc_lib", - ":cc_test_protos", "//external:gtest_main", ], ) + +################################################################################ +# Java support +################################################################################ +genrule( + name = "generate_java_descriptor_proto", + srcs = ["src/google/protobuf/descriptor.proto"], + outs = ["com/google/protobuf/DescriptorProtos.java"], + cmd = "$(location :protoc) --java_out=$(@D)/../../.. $<", + tools = [":protoc"], +) + +java_library( + name = "java_proto", + srcs = glob([ + "java/src/main/java/com/google/protobuf/*.java", + ]) + [ + ":generate_java_descriptor_proto", + ], + visibility = ["//visibility:public"], +) + +################################################################################ +# Python support +################################################################################ + +copied_srcs( + name = "python_srcs", + srcs = glob( + [ + "python/google/protobuf/*.py", + "python/google/protobuf/**/*.py", + ], + exclude = [ + "python/google/protobuf/internal/*_test.py", + "python/google/protobuf/internal/test_util.py", + ], + ), + include = "python", +) + +py_proto_library( + name = "python_proto", + srcs = WELL_KNOWN_PROTOS, + include = "src", + py_extra_srcs = [":python_srcs"], + visibility = ["//visibility:public"], +) + +copied_srcs( + name = "python_test_srcs", + srcs = glob( + [ + "python/google/protobuf/internal/*_test.py", + "python/google/protobuf/internal/test_util.py", + ], + ), + include = "python", +) + +py_proto_library( + name = "python_common_test_protos", + srcs = LITE_TEST_PROTOS + TEST_PROTOS, + include = "src", + deps = [":python_proto"], +) + +py_proto_library( + name = "python_specific_test_protos", + srcs = glob(["python/google/protobuf/internal/*.proto"]), + include = "python", + deps = [":python_common_test_protos"], +) + +py_library( + name = "python_tests", + srcs = [":python_test_srcs"], + deps = [ + ":python_common_test_protos", + ":python_proto", + ":python_specific_test_protos", + ], +) + +internal_protobuf_py_tests( + name = "python_tests", + modules = [ + "descriptor_database_test", + "descriptor_pool_test", + "descriptor_test", + "generator_test", + "json_format_test", + "message_factory_test", + # "message_test", # failed due to testdata path + "proto_builder_test", + # "reflection_test", # failed due to testdata path + "service_reflection_test", + "symbol_database_test", + "text_encoding_test", + # "text_format_test", # failed due to testdata path + "unknown_fields_test", + "wire_format_test", + ], + deps = [":python_tests"], +) -- cgit v1.2.3 From 7b948cc7c5da776aad7f0008e2e108ffba961f37 Mon Sep 17 00:00:00 2001 From: Jisi Liu Date: Mon, 19 Oct 2015 17:56:27 -0700 Subject: Support python for bazel. --- BUILD | 15 +++++++++++++++ protobuf.bzl | 42 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 54 insertions(+), 3 deletions(-) (limited to 'BUILD') diff --git a/BUILD b/BUILD index 465628fa..281436a3 100644 --- a/BUILD +++ b/BUILD @@ -463,6 +463,21 @@ java_library( # Python support ################################################################################ +# Hack: +# protoc generated files contain imports like: +# "from google.protobuf.xxx import yyy" +# However, the sources files of the python runtime are not directly under +# "google/protobuf" (they are under python/google/protobuf). We workaround +# this by copying runtime source files into the desired location to workaround +# the import issue. Ideally py_library should support something similiar to the +# "include" attribute in cc_library to inject the PYTHON_PATH for all libraries +# that depend on the target. +# +# If you use python protobuf as a third_party library in your bazel managed +# project, please import the whole package to //google/protobuf in your +# project. Otherwise, bazel disallows generated files out of the current +# package, thus we won't be able to copy protobuf runtime files into +# //google/protobuf/. copied_srcs( name = "python_srcs", srcs = glob( diff --git a/protobuf.bzl b/protobuf.bzl index 87aed9c8..27e88850 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -160,6 +160,15 @@ def copied_srcs( srcs, include, **kargs): + """Bazel rule to fix sources file to workaround with python path issues. + + Args: + name: the name of the copied_srcs rule, which will be the name of the + generated filegroup. + srcs: the source files to be copied. + include: the expected import root of the source. + **kargs: extra arguments that will be passed into the filegroup. + """ outs = [_RelativeOutputPath(s, include) for s in srcs] native.genrule( @@ -185,6 +194,21 @@ def py_proto_library( include=None, protoc=":protoc", **kargs): + """Bazel rule to create a Python protobuf library from proto source files + + Args: + name: the name of the py_proto_library. + srcs: the .proto files of the py_proto_library. + deps: a list of dependency labels; must be py_proto_library. + py_libs: a list of other py_library targets depended by the generated + py_library. + py_extra_srcs: extra source files that will be added to the output + py_library. This attribute is used for internal bootstrapping. + include: a string indicating the include path of the .proto files. + protoc: the label of the protocol compiler to generate the sources. + **kargs: other keyword arguments that are passed to cc_library. + + """ outs = _PyOuts(srcs) _proto_gen( name=name + "_genproto", @@ -196,8 +220,9 @@ def py_proto_library( outs=outs, ) - copied_srcs_name=name + "_copied_srcs" if include != None: + # Copy the output files to the desired location to make the import work. + copied_srcs_name=name + "_copied_srcs" copied_srcs( name=copied_srcs_name, srcs=outs, @@ -214,9 +239,20 @@ def internal_protobuf_py_tests( name, modules=[], **kargs): + """Bazel rules to create batch tests for protobuf internal. + + Args: + name: the name of the rule. + modules: a list of modules for tests. The macro will create a py_test for + each of the parameter with the source "google/protobuf/%s.py" + kargs: extra parameters that will be passed into the py_test. + + """ for m in modules: + s = _RelativeOutputPath( + "python/google/protobuf/internal/%s.py" % m, "python") native.py_test( name="py_%s" % m, - srcs=["google/protobuf/internal/%s.py" % m], - main="google/protobuf/internal/%s.py" % m, + srcs=[s], + main=s, **kargs) -- cgit v1.2.3 From 04658a3c24e1f4b7e1142843bb0a33d55e4f821f Mon Sep 17 00:00:00 2001 From: Jisi Liu Date: Tue, 20 Oct 2015 15:00:13 -0700 Subject: Change default value of protoc on xx_proto_library rules. --- BUILD | 14 ++++++++++++-- protobuf.bzl | 4 ++-- 2 files changed, 14 insertions(+), 4 deletions(-) (limited to 'BUILD') diff --git a/BUILD b/BUILD index 281436a3..4ccf9a6e 100644 --- a/BUILD +++ b/BUILD @@ -18,8 +18,13 @@ COPTS = [ # Bazel should provide portable link_opts for pthread. LINK_OPTS = ["-lpthread"] -load("protobuf", "cc_proto_library", "py_proto_library", "copied_srcs", - "internal_protobuf_py_tests") +load( + "protobuf", + "cc_proto_library", + "py_proto_library", + "copied_srcs", + "internal_protobuf_py_tests", +) cc_library( name = "protobuf_lite", @@ -151,6 +156,7 @@ cc_proto_library( include = "src", cc_libs = [":protobuf"], internal_bootstrap_hack = 1, + protoc = ":protoc", ) ################################################################################ @@ -327,6 +333,7 @@ cc_proto_library( name = "cc_test_protos", srcs = LITE_TEST_PROTOS + TEST_PROTOS, include = "src", + protoc = ":protoc", deps = [":cc_wkt_protos"], ) @@ -497,6 +504,7 @@ py_proto_library( name = "python_proto", srcs = WELL_KNOWN_PROTOS, include = "src", + protoc = ":protoc", py_extra_srcs = [":python_srcs"], visibility = ["//visibility:public"], ) @@ -516,6 +524,7 @@ py_proto_library( name = "python_common_test_protos", srcs = LITE_TEST_PROTOS + TEST_PROTOS, include = "src", + protoc = ":protoc", deps = [":python_proto"], ) @@ -523,6 +532,7 @@ py_proto_library( name = "python_specific_test_protos", srcs = glob(["python/google/protobuf/internal/*.proto"]), include = "python", + protoc = ":protoc", deps = [":python_common_test_protos"], ) diff --git a/protobuf.bzl b/protobuf.bzl index 27e88850..c7d66086 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -95,7 +95,7 @@ def cc_proto_library( deps=[], cc_libs=[], include=None, - protoc=":protoc", + protoc="//google/protobuf:protoc", internal_bootstrap_hack=False, **kargs): """Bazel rule to create a C++ protobuf library from proto source files @@ -192,7 +192,7 @@ def py_proto_library( py_libs=[], py_extra_srcs=[], include=None, - protoc=":protoc", + protoc="//google/protobuf:protoc", **kargs): """Bazel rule to create a Python protobuf library from proto source files -- cgit v1.2.3 From 53a56be4c49174bc2697d648b3d41ff141fee1d7 Mon Sep 17 00:00:00 2001 From: Jisi Liu Date: Tue, 20 Oct 2015 15:18:20 -0700 Subject: Change the impl rule include to includes. We need to use the list to indicate field presense. The field must only contain 0 or 1 string element. --- BUILD | 14 +++++++------- protobuf.bzl | 34 ++++++++++++++++++++++------------ 2 files changed, 29 insertions(+), 19 deletions(-) (limited to 'BUILD') diff --git a/BUILD b/BUILD index 4ccf9a6e..7f78a409 100644 --- a/BUILD +++ b/BUILD @@ -124,13 +124,13 @@ cc_library( deps = [":protobuf_lite"], ) -objc_library( - name = "protobuf_objc", - hdrs = ["objectivec/GPBProtocolBuffers.h"], - includes = ["objectivec"], - non_arc_srcs = ["objectivec/GPBProtocolBuffers.m"], - visibility = ["//visibility:public"], -) +# objc_library( +# name = "protobuf_objc", +# hdrs = ["objectivec/GPBProtocolBuffers.h"], +# includes = ["objectivec"], +# non_arc_srcs = ["objectivec/GPBProtocolBuffers.m"], +# visibility = ["//visibility:public"], +# ) RELATIVE_WELL_KNOWN_PROTOS = [ # AUTOGEN(well_known_protos) diff --git a/protobuf.bzl b/protobuf.bzl index c7d66086..1fc20dda 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -1,13 +1,13 @@ # -*- mode: python; -*- PYTHON-PREPROCESSING-REQUIRED def _GenDir(ctx): - if ctx.attr.include == None: + if not ctx.attr.includes: return "" - if not ctx.attr.include: + if not ctx.attr.includes[0]: return ctx.label.package if not ctx.label.package: - return ctx.attr.include - return ctx.label.package + '/' + ctx.attr.include + return ctx.attr.includes[0] + return ctx.label.package + '/' + ctx.attr.includes[0] def _CcOuts(srcs): return [s[:-len(".proto")] + ".pb.h" for s in srcs] + \ @@ -44,7 +44,11 @@ def _proto_gen_impl(ctx): deps = [] deps += ctx.files.srcs gen_dir = _GenDir(ctx) - import_flags = ["-I" + gen_dir] + if gen_dir: + import_flags = ["-I" + gen_dir] + else: + import_flags = ["-I."] + for dep in ctx.attr.deps: import_flags += dep.proto.import_flags deps += dep.proto.deps @@ -75,7 +79,7 @@ _proto_gen = rule( attrs = { "srcs": attr.label_list(allow_files = True), "deps": attr.label_list(providers = ["proto"]), - "include": attr.string(), + "includes": attr.string_list(), "protoc": attr.label( executable = True, single_file = True, @@ -116,6 +120,10 @@ def cc_proto_library( """ + includes = [] + if include != None: + includes = [include] + if internal_bootstrap_hack: # For pre-checked-in generated files, we add the internal_bootstrap_hack # which will skip the codegen action. @@ -123,7 +131,7 @@ def cc_proto_library( name=name + "_genproto", srcs=srcs, deps=[s + "_genproto" for s in deps], - include=include, + includes=includes, protoc=protoc, ) # An empty cc_library to make rule dependency consistent. @@ -137,15 +145,12 @@ def cc_proto_library( name=name + "_genproto", srcs=srcs, deps=[s + "_genproto" for s in deps], - include=include, + includes=includes, protoc=protoc, gen_cc=1, outs=outs, ) - includes = [] - if include != None: - includes = [include] native.cc_library( name=name, @@ -210,11 +215,16 @@ def py_proto_library( """ outs = _PyOuts(srcs) + + includes = [] + if include != None: + includes = [include] + _proto_gen( name=name + "_genproto", srcs=srcs, deps=[s + "_genproto" for s in deps], - include=include, + includes=includes, protoc=protoc, gen_py=1, outs=outs, -- cgit v1.2.3 From 14c8f8ac1df72da456ea2d30b42b67199930e3ab Mon Sep 17 00:00:00 2001 From: Jisi Liu Date: Tue, 20 Oct 2015 15:36:22 -0700 Subject: Uncomment objc tests --- BUILD | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'BUILD') diff --git a/BUILD b/BUILD index 7f78a409..a3d8a8f6 100644 --- a/BUILD +++ b/BUILD @@ -124,13 +124,13 @@ cc_library( deps = [":protobuf_lite"], ) -# objc_library( -# name = "protobuf_objc", -# hdrs = ["objectivec/GPBProtocolBuffers.h"], -# includes = ["objectivec"], -# non_arc_srcs = ["objectivec/GPBProtocolBuffers.m"], -# visibility = ["//visibility:public"], -# ) +objc_library( + name = "protobuf_objc", + hdrs = ["objectivec/GPBProtocolBuffers.h"], + includes = ["objectivec"], + non_arc_srcs = ["objectivec/GPBProtocolBuffers.m"], + visibility = ["//visibility:public"], +) RELATIVE_WELL_KNOWN_PROTOS = [ # AUTOGEN(well_known_protos) @@ -470,6 +470,8 @@ java_library( # Python support ################################################################################ +# Requires: six for python 2/3 compatibility. `pip install six` + # Hack: # protoc generated files contain imports like: # "from google.protobuf.xxx import yyy" -- cgit v1.2.3 From bc4fd15209d5f874a2d9176761f46cfe5af33ed4 Mon Sep 17 00:00:00 2001 From: Jisi Liu Date: Tue, 20 Oct 2015 16:02:58 -0700 Subject: Rename copeid_src to internal_copied_filegroup --- BUILD | 6 +++--- protobuf.bzl | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) (limited to 'BUILD') diff --git a/BUILD b/BUILD index a3d8a8f6..9636497d 100644 --- a/BUILD +++ b/BUILD @@ -22,7 +22,7 @@ load( "protobuf", "cc_proto_library", "py_proto_library", - "copied_srcs", + "internal_copied_filegroup", "internal_protobuf_py_tests", ) @@ -487,7 +487,7 @@ java_library( # project. Otherwise, bazel disallows generated files out of the current # package, thus we won't be able to copy protobuf runtime files into # //google/protobuf/. -copied_srcs( +internal_copied_filegroup( name = "python_srcs", srcs = glob( [ @@ -511,7 +511,7 @@ py_proto_library( visibility = ["//visibility:public"], ) -copied_srcs( +internal_copied_filegroup( name = "python_test_srcs", srcs = glob( [ diff --git a/protobuf.bzl b/protobuf.bzl index 3b525815..2199caf1 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -160,7 +160,7 @@ def cc_proto_library( **kargs) -def copied_srcs( +def internal_copied_filegroup( name, srcs, include, @@ -168,8 +168,8 @@ def copied_srcs( """Bazel rule to fix sources file to workaround with python path issues. Args: - name: the name of the copied_srcs rule, which will be the name of the - generated filegroup. + name: the name of the internal_copied_filegroup rule, which will be the + name of the generated filegroup. srcs: the source files to be copied. include: the expected import root of the source. **kargs: extra arguments that will be passed into the filegroup. @@ -232,12 +232,12 @@ def py_proto_library( if include != None: # Copy the output files to the desired location to make the import work. - copied_srcs_name=name + "_copied_srcs" - copied_srcs( - name=copied_srcs_name, + internal_copied_filegroup_name=name + "_internal_copied_filegroup" + internal_copied_filegroup( + name=internal_copied_filegroup_name, srcs=outs, include=include) - outs=[copied_srcs_name] + outs=[internal_copied_filegroup_name] native.py_library( name=name, -- cgit v1.2.3 From 8f54026ded729f4225c35e13f635747edbc580eb Mon Sep 17 00:00:00 2001 From: Jisi Liu Date: Tue, 20 Oct 2015 16:21:41 -0700 Subject: avoid name duplication. --- BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'BUILD') diff --git a/BUILD b/BUILD index 9636497d..11f00b81 100644 --- a/BUILD +++ b/BUILD @@ -549,7 +549,7 @@ py_library( ) internal_protobuf_py_tests( - name = "python_tests", + name = "python_tests_batch", modules = [ "descriptor_database_test", "descriptor_pool_test", -- cgit v1.2.3