From 993fb7013e0139ec7b16abcada7995f3a642e331 Mon Sep 17 00:00:00 2001 From: Jisi Liu Date: Mon, 19 Oct 2015 17:19:49 -0700 Subject: Python bazel support. --- protobuf.bzl | 99 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 94 insertions(+), 5 deletions(-) (limited to 'protobuf.bzl') diff --git a/protobuf.bzl b/protobuf.bzl index b83f7f5a..87aed9c8 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -1,6 +1,6 @@ # -*- mode: python; -*- PYTHON-PREPROCESSING-REQUIRED -def _gen_dir(ctx): +def _GenDir(ctx): if ctx.attr.include == None: return "" if not ctx.attr.include: @@ -9,19 +9,41 @@ def _gen_dir(ctx): return ctx.attr.include return ctx.label.package + '/' + ctx.attr.include -def _cc_outs(srcs): +def _CcOuts(srcs): return [s[:-len(".proto")] + ".pb.h" for s in srcs] + \ [s[:-len(".proto")] + ".pb.cc" for s in srcs] -def _py_outs(srcs): +def _PyOuts(srcs): return [s[:-len(".proto")] + "_pb2.py" for s in srcs] +def _RelativeOutputPath(path, include): + if include == None: + return path + + if not path.startswith(include): + fail("Include path %s isn't part of the path %s." % (include, path)) + + if include and include[-1] != '/': + include = include + '/' + + path = path[len(include):] + + if not path.startswith(PACKAGE_NAME): + fail("The package %s is not within the path %s" % (PACKAGE_NAME, path)) + + if not PACKAGE_NAME: + return path + + return path[len(PACKAGE_NAME)+1:] + + + def _proto_gen_impl(ctx): """General implementation for generating protos""" srcs = ctx.files.srcs deps = [] deps += ctx.files.srcs - gen_dir = _gen_dir(ctx) + gen_dir = _GenDir(ctx) import_flags = ["-I" + gen_dir] for dep in ctx.attr.deps: import_flags += dep.proto.import_flags @@ -110,7 +132,7 @@ def cc_proto_library( **kargs) return - outs = _cc_outs(srcs) + outs = _CcOuts(srcs) _proto_gen( name=name + "_genproto", srcs=srcs, @@ -131,3 +153,70 @@ def cc_proto_library( deps=cc_libs + deps, includes=includes, **kargs) + + +def copied_srcs( + name, + srcs, + include, + **kargs): + outs = [_RelativeOutputPath(s, include) for s in srcs] + + native.genrule( + name=name+"_genrule", + srcs=srcs, + outs=outs, + cmd=";".join(["cp $(location %s) $(location %s)" % \ + (s, _RelativeOutputPath(s, include)) \ + for s in srcs])) + + native.filegroup( + name=name, + srcs=outs, + **kargs) + + +def py_proto_library( + name, + srcs=[], + deps=[], + py_libs=[], + py_extra_srcs=[], + include=None, + protoc=":protoc", + **kargs): + outs = _PyOuts(srcs) + _proto_gen( + name=name + "_genproto", + srcs=srcs, + deps=[s + "_genproto" for s in deps], + include=include, + protoc=protoc, + gen_py=1, + outs=outs, + ) + + copied_srcs_name=name + "_copied_srcs" + if include != None: + copied_srcs( + name=copied_srcs_name, + srcs=outs, + include=include) + srcs=[copied_srcs_name] + + native.py_library( + name=name, + srcs=srcs+py_extra_srcs, + deps=py_libs, + **kargs) + +def internal_protobuf_py_tests( + name, + modules=[], + **kargs): + for m in modules: + native.py_test( + name="py_%s" % m, + srcs=["google/protobuf/internal/%s.py" % m], + main="google/protobuf/internal/%s.py" % m, + **kargs) -- cgit v1.2.3 From 7b948cc7c5da776aad7f0008e2e108ffba961f37 Mon Sep 17 00:00:00 2001 From: Jisi Liu Date: Mon, 19 Oct 2015 17:56:27 -0700 Subject: Support python for bazel. --- BUILD | 15 +++++++++++++++ protobuf.bzl | 42 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 54 insertions(+), 3 deletions(-) (limited to 'protobuf.bzl') diff --git a/BUILD b/BUILD index 465628fa..281436a3 100644 --- a/BUILD +++ b/BUILD @@ -463,6 +463,21 @@ java_library( # Python support ################################################################################ +# Hack: +# protoc generated files contain imports like: +# "from google.protobuf.xxx import yyy" +# However, the sources files of the python runtime are not directly under +# "google/protobuf" (they are under python/google/protobuf). We workaround +# this by copying runtime source files into the desired location to workaround +# the import issue. Ideally py_library should support something similiar to the +# "include" attribute in cc_library to inject the PYTHON_PATH for all libraries +# that depend on the target. +# +# If you use python protobuf as a third_party library in your bazel managed +# project, please import the whole package to //google/protobuf in your +# project. Otherwise, bazel disallows generated files out of the current +# package, thus we won't be able to copy protobuf runtime files into +# //google/protobuf/. copied_srcs( name = "python_srcs", srcs = glob( diff --git a/protobuf.bzl b/protobuf.bzl index 87aed9c8..27e88850 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -160,6 +160,15 @@ def copied_srcs( srcs, include, **kargs): + """Bazel rule to fix sources file to workaround with python path issues. + + Args: + name: the name of the copied_srcs rule, which will be the name of the + generated filegroup. + srcs: the source files to be copied. + include: the expected import root of the source. + **kargs: extra arguments that will be passed into the filegroup. + """ outs = [_RelativeOutputPath(s, include) for s in srcs] native.genrule( @@ -185,6 +194,21 @@ def py_proto_library( include=None, protoc=":protoc", **kargs): + """Bazel rule to create a Python protobuf library from proto source files + + Args: + name: the name of the py_proto_library. + srcs: the .proto files of the py_proto_library. + deps: a list of dependency labels; must be py_proto_library. + py_libs: a list of other py_library targets depended by the generated + py_library. + py_extra_srcs: extra source files that will be added to the output + py_library. This attribute is used for internal bootstrapping. + include: a string indicating the include path of the .proto files. + protoc: the label of the protocol compiler to generate the sources. + **kargs: other keyword arguments that are passed to cc_library. + + """ outs = _PyOuts(srcs) _proto_gen( name=name + "_genproto", @@ -196,8 +220,9 @@ def py_proto_library( outs=outs, ) - copied_srcs_name=name + "_copied_srcs" if include != None: + # Copy the output files to the desired location to make the import work. + copied_srcs_name=name + "_copied_srcs" copied_srcs( name=copied_srcs_name, srcs=outs, @@ -214,9 +239,20 @@ def internal_protobuf_py_tests( name, modules=[], **kargs): + """Bazel rules to create batch tests for protobuf internal. + + Args: + name: the name of the rule. + modules: a list of modules for tests. The macro will create a py_test for + each of the parameter with the source "google/protobuf/%s.py" + kargs: extra parameters that will be passed into the py_test. + + """ for m in modules: + s = _RelativeOutputPath( + "python/google/protobuf/internal/%s.py" % m, "python") native.py_test( name="py_%s" % m, - srcs=["google/protobuf/internal/%s.py" % m], - main="google/protobuf/internal/%s.py" % m, + srcs=[s], + main=s, **kargs) -- cgit v1.2.3 From 04658a3c24e1f4b7e1142843bb0a33d55e4f821f Mon Sep 17 00:00:00 2001 From: Jisi Liu Date: Tue, 20 Oct 2015 15:00:13 -0700 Subject: Change default value of protoc on xx_proto_library rules. --- BUILD | 14 ++++++++++++-- protobuf.bzl | 4 ++-- 2 files changed, 14 insertions(+), 4 deletions(-) (limited to 'protobuf.bzl') diff --git a/BUILD b/BUILD index 281436a3..4ccf9a6e 100644 --- a/BUILD +++ b/BUILD @@ -18,8 +18,13 @@ COPTS = [ # Bazel should provide portable link_opts for pthread. LINK_OPTS = ["-lpthread"] -load("protobuf", "cc_proto_library", "py_proto_library", "copied_srcs", - "internal_protobuf_py_tests") +load( + "protobuf", + "cc_proto_library", + "py_proto_library", + "copied_srcs", + "internal_protobuf_py_tests", +) cc_library( name = "protobuf_lite", @@ -151,6 +156,7 @@ cc_proto_library( include = "src", cc_libs = [":protobuf"], internal_bootstrap_hack = 1, + protoc = ":protoc", ) ################################################################################ @@ -327,6 +333,7 @@ cc_proto_library( name = "cc_test_protos", srcs = LITE_TEST_PROTOS + TEST_PROTOS, include = "src", + protoc = ":protoc", deps = [":cc_wkt_protos"], ) @@ -497,6 +504,7 @@ py_proto_library( name = "python_proto", srcs = WELL_KNOWN_PROTOS, include = "src", + protoc = ":protoc", py_extra_srcs = [":python_srcs"], visibility = ["//visibility:public"], ) @@ -516,6 +524,7 @@ py_proto_library( name = "python_common_test_protos", srcs = LITE_TEST_PROTOS + TEST_PROTOS, include = "src", + protoc = ":protoc", deps = [":python_proto"], ) @@ -523,6 +532,7 @@ py_proto_library( name = "python_specific_test_protos", srcs = glob(["python/google/protobuf/internal/*.proto"]), include = "python", + protoc = ":protoc", deps = [":python_common_test_protos"], ) diff --git a/protobuf.bzl b/protobuf.bzl index 27e88850..c7d66086 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -95,7 +95,7 @@ def cc_proto_library( deps=[], cc_libs=[], include=None, - protoc=":protoc", + protoc="//google/protobuf:protoc", internal_bootstrap_hack=False, **kargs): """Bazel rule to create a C++ protobuf library from proto source files @@ -192,7 +192,7 @@ def py_proto_library( py_libs=[], py_extra_srcs=[], include=None, - protoc=":protoc", + protoc="//google/protobuf:protoc", **kargs): """Bazel rule to create a Python protobuf library from proto source files -- cgit v1.2.3 From 53a56be4c49174bc2697d648b3d41ff141fee1d7 Mon Sep 17 00:00:00 2001 From: Jisi Liu Date: Tue, 20 Oct 2015 15:18:20 -0700 Subject: Change the impl rule include to includes. We need to use the list to indicate field presense. The field must only contain 0 or 1 string element. --- BUILD | 14 +++++++------- protobuf.bzl | 34 ++++++++++++++++++++++------------ 2 files changed, 29 insertions(+), 19 deletions(-) (limited to 'protobuf.bzl') diff --git a/BUILD b/BUILD index 4ccf9a6e..7f78a409 100644 --- a/BUILD +++ b/BUILD @@ -124,13 +124,13 @@ cc_library( deps = [":protobuf_lite"], ) -objc_library( - name = "protobuf_objc", - hdrs = ["objectivec/GPBProtocolBuffers.h"], - includes = ["objectivec"], - non_arc_srcs = ["objectivec/GPBProtocolBuffers.m"], - visibility = ["//visibility:public"], -) +# objc_library( +# name = "protobuf_objc", +# hdrs = ["objectivec/GPBProtocolBuffers.h"], +# includes = ["objectivec"], +# non_arc_srcs = ["objectivec/GPBProtocolBuffers.m"], +# visibility = ["//visibility:public"], +# ) RELATIVE_WELL_KNOWN_PROTOS = [ # AUTOGEN(well_known_protos) diff --git a/protobuf.bzl b/protobuf.bzl index c7d66086..1fc20dda 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -1,13 +1,13 @@ # -*- mode: python; -*- PYTHON-PREPROCESSING-REQUIRED def _GenDir(ctx): - if ctx.attr.include == None: + if not ctx.attr.includes: return "" - if not ctx.attr.include: + if not ctx.attr.includes[0]: return ctx.label.package if not ctx.label.package: - return ctx.attr.include - return ctx.label.package + '/' + ctx.attr.include + return ctx.attr.includes[0] + return ctx.label.package + '/' + ctx.attr.includes[0] def _CcOuts(srcs): return [s[:-len(".proto")] + ".pb.h" for s in srcs] + \ @@ -44,7 +44,11 @@ def _proto_gen_impl(ctx): deps = [] deps += ctx.files.srcs gen_dir = _GenDir(ctx) - import_flags = ["-I" + gen_dir] + if gen_dir: + import_flags = ["-I" + gen_dir] + else: + import_flags = ["-I."] + for dep in ctx.attr.deps: import_flags += dep.proto.import_flags deps += dep.proto.deps @@ -75,7 +79,7 @@ _proto_gen = rule( attrs = { "srcs": attr.label_list(allow_files = True), "deps": attr.label_list(providers = ["proto"]), - "include": attr.string(), + "includes": attr.string_list(), "protoc": attr.label( executable = True, single_file = True, @@ -116,6 +120,10 @@ def cc_proto_library( """ + includes = [] + if include != None: + includes = [include] + if internal_bootstrap_hack: # For pre-checked-in generated files, we add the internal_bootstrap_hack # which will skip the codegen action. @@ -123,7 +131,7 @@ def cc_proto_library( name=name + "_genproto", srcs=srcs, deps=[s + "_genproto" for s in deps], - include=include, + includes=includes, protoc=protoc, ) # An empty cc_library to make rule dependency consistent. @@ -137,15 +145,12 @@ def cc_proto_library( name=name + "_genproto", srcs=srcs, deps=[s + "_genproto" for s in deps], - include=include, + includes=includes, protoc=protoc, gen_cc=1, outs=outs, ) - includes = [] - if include != None: - includes = [include] native.cc_library( name=name, @@ -210,11 +215,16 @@ def py_proto_library( """ outs = _PyOuts(srcs) + + includes = [] + if include != None: + includes = [include] + _proto_gen( name=name + "_genproto", srcs=srcs, deps=[s + "_genproto" for s in deps], - include=include, + includes=includes, protoc=protoc, gen_py=1, outs=outs, -- cgit v1.2.3 From a33fa8eddc05cb48eea940fbbdf5dce24d9cfa49 Mon Sep 17 00:00:00 2001 From: Jisi Liu Date: Tue, 20 Oct 2015 15:30:44 -0700 Subject: fix sources for python target and add needed dependencies. --- protobuf.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'protobuf.bzl') diff --git a/protobuf.bzl b/protobuf.bzl index 1fc20dda..3b525815 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -237,12 +237,12 @@ def py_proto_library( name=copied_srcs_name, srcs=outs, include=include) - srcs=[copied_srcs_name] + outs=[copied_srcs_name] native.py_library( name=name, - srcs=srcs+py_extra_srcs, - deps=py_libs, + srcs=outs+py_extra_srcs, + deps=py_libs+deps, **kargs) def internal_protobuf_py_tests( -- cgit v1.2.3 From bc4fd15209d5f874a2d9176761f46cfe5af33ed4 Mon Sep 17 00:00:00 2001 From: Jisi Liu Date: Tue, 20 Oct 2015 16:02:58 -0700 Subject: Rename copeid_src to internal_copied_filegroup --- BUILD | 6 +++--- protobuf.bzl | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) (limited to 'protobuf.bzl') diff --git a/BUILD b/BUILD index a3d8a8f6..9636497d 100644 --- a/BUILD +++ b/BUILD @@ -22,7 +22,7 @@ load( "protobuf", "cc_proto_library", "py_proto_library", - "copied_srcs", + "internal_copied_filegroup", "internal_protobuf_py_tests", ) @@ -487,7 +487,7 @@ java_library( # project. Otherwise, bazel disallows generated files out of the current # package, thus we won't be able to copy protobuf runtime files into # //google/protobuf/. -copied_srcs( +internal_copied_filegroup( name = "python_srcs", srcs = glob( [ @@ -511,7 +511,7 @@ py_proto_library( visibility = ["//visibility:public"], ) -copied_srcs( +internal_copied_filegroup( name = "python_test_srcs", srcs = glob( [ diff --git a/protobuf.bzl b/protobuf.bzl index 3b525815..2199caf1 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -160,7 +160,7 @@ def cc_proto_library( **kargs) -def copied_srcs( +def internal_copied_filegroup( name, srcs, include, @@ -168,8 +168,8 @@ def copied_srcs( """Bazel rule to fix sources file to workaround with python path issues. Args: - name: the name of the copied_srcs rule, which will be the name of the - generated filegroup. + name: the name of the internal_copied_filegroup rule, which will be the + name of the generated filegroup. srcs: the source files to be copied. include: the expected import root of the source. **kargs: extra arguments that will be passed into the filegroup. @@ -232,12 +232,12 @@ def py_proto_library( if include != None: # Copy the output files to the desired location to make the import work. - copied_srcs_name=name + "_copied_srcs" - copied_srcs( - name=copied_srcs_name, + internal_copied_filegroup_name=name + "_internal_copied_filegroup" + internal_copied_filegroup( + name=internal_copied_filegroup_name, srcs=outs, include=include) - outs=[copied_srcs_name] + outs=[internal_copied_filegroup_name] native.py_library( name=name, -- cgit v1.2.3