Merge pull request #867 from pherl/master

Python bazel support

2 files changed, 299 insertions, 46 deletions

diff --git a/BUILD b/BUILD
index 69ea6cf3..11f00b81 100644
--- a/BUILD
+++ b/BUILD
@@ -18,7 +18,13 @@ COPTS = [
 # Bazel should provide portable link_opts for pthread.
 LINK_OPTS = ["-lpthread"]
 
-load("protobuf", "cc_proto_library")
+load(
+    "protobuf",
+    "cc_proto_library",
+    "py_proto_library",
+    "internal_copied_filegroup",
+    "internal_protobuf_py_tests",
+)
 
 cc_library(
     name = "protobuf_lite",
@@ -126,7 +132,7 @@ objc_library(
     visibility = ["//visibility:public"],
 )
 
-WELL_KNOWN_PROTOS = [
+RELATIVE_WELL_KNOWN_PROTOS = [
     # AUTOGEN(well_known_protos)
     "google/protobuf/any.proto",
     "google/protobuf/api.proto",
@@ -142,12 +148,15 @@ WELL_KNOWN_PROTOS = [
     "google/protobuf/wrappers.proto",
 ]
 
+WELL_KNOWN_PROTOS = ["src/" + s for s in RELATIVE_WELL_KNOWN_PROTOS]
+
 cc_proto_library(
     name = "cc_wkt_protos",
-    srcs = ["src/" + s for s in WELL_KNOWN_PROTOS],
-    internal_bootstrap_hack = 1,
+    srcs = WELL_KNOWN_PROTOS,
     include = "src",
     cc_libs = [":protobuf"],
+    internal_bootstrap_hack = 1,
+    protoc = ":protoc",
 )
 
 ################################################################################
@@ -264,31 +273,10 @@ cc_binary(
 )
 
 ################################################################################
-# Java support
-################################################################################
-genrule(
-    name = "generate_java_descriptor_proto",
-    srcs = ["src/google/protobuf/descriptor.proto"],
-    outs = ["com/google/protobuf/DescriptorProtos.java"],
-    cmd = "$(location :protoc) --java_out=$(@D)/../../.. $<",
-    tools = [":protoc"],
-)
-
-java_library(
-    name = "java_proto",
-    srcs = glob([
-        "java/src/main/java/com/google/protobuf/*.java",
-    ]) + [
-        ":generate_java_descriptor_proto",
-    ],
-    visibility = ["//visibility:public"],
-)
-
-################################################################################
 # Tests
 ################################################################################
 
-LITE_TEST_PROTOS = [
+RELATIVE_LITE_TEST_PROTOS = [
     # AUTOGEN(lite_test_protos)
     "google/protobuf/map_lite_unittest.proto",
     "google/protobuf/unittest_import_lite.proto",
@@ -297,7 +285,9 @@ LITE_TEST_PROTOS = [
     "google/protobuf/unittest_no_arena_lite.proto",
 ]
 
-TEST_PROTOS = [
+LITE_TEST_PROTOS = ["src/" + s for s in RELATIVE_LITE_TEST_PROTOS]
+
+RELATIVE_TEST_PROTOS = [
     # AUTOGEN(test_protos)
     "google/protobuf/any_test.proto",
     "google/protobuf/compiler/cpp/cpp_test_bad_identifiers.proto",
@@ -337,10 +327,13 @@ TEST_PROTOS = [
     "google/protobuf/util/json_format_proto3.proto",
 ]
 
+TEST_PROTOS = ["src/" + s for s in RELATIVE_TEST_PROTOS]
+
 cc_proto_library(
     name = "cc_test_protos",
-    srcs = ["src/" + s for s in (LITE_TEST_PROTOS + TEST_PROTOS)],
+    srcs = LITE_TEST_PROTOS + TEST_PROTOS,
     include = "src",
+    protoc = ":protoc",
     deps = [":cc_wkt_protos"],
 )
 
@@ -445,9 +438,134 @@ cc_test(
     ],
     linkopts = LINK_OPTS,
     deps = [
+        ":cc_test_protos",
         ":protobuf",
         ":protoc_lib",
-        ":cc_test_protos",
         "//external:gtest_main",
     ],
 )
+
+################################################################################
+# Java support
+################################################################################
+genrule(
+    name = "generate_java_descriptor_proto",
+    srcs = ["src/google/protobuf/descriptor.proto"],
+    outs = ["com/google/protobuf/DescriptorProtos.java"],
+    cmd = "$(location :protoc) --java_out=$(@D)/../../.. $<",
+    tools = [":protoc"],
+)
+
+java_library(
+    name = "java_proto",
+    srcs = glob([
+        "java/src/main/java/com/google/protobuf/*.java",
+    ]) + [
+        ":generate_java_descriptor_proto",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+################################################################################
+# Python support
+################################################################################
+
+# Requires: six for python 2/3 compatibility.  `pip install six`
+
+# Hack:
+# protoc generated files contain imports like:
+#   "from google.protobuf.xxx import yyy"
+# However, the sources files of the python runtime are not directly under
+# "google/protobuf" (they are under python/google/protobuf).  We workaround
+# this by copying runtime source files into the desired location to workaround
+# the import issue. Ideally py_library should support something similiar to the
+# "include" attribute in cc_library to inject the PYTHON_PATH for all libraries
+# that depend on the target.
+#
+# If you use python protobuf as a third_party library in your bazel managed
+# project, please import the whole package to //google/protobuf in your
+# project. Otherwise, bazel disallows generated files out of the current
+# package, thus we won't be able to copy protobuf runtime files into
+# //google/protobuf/.
+internal_copied_filegroup(
+    name = "python_srcs",
+    srcs = glob(
+        [
+            "python/google/protobuf/*.py",
+            "python/google/protobuf/**/*.py",
+        ],
+        exclude = [
+            "python/google/protobuf/internal/*_test.py",
+            "python/google/protobuf/internal/test_util.py",
+        ],
+    ),
+    include = "python",
+)
+
+py_proto_library(
+    name = "python_proto",
+    srcs = WELL_KNOWN_PROTOS,
+    include = "src",
+    protoc = ":protoc",
+    py_extra_srcs = [":python_srcs"],
+    visibility = ["//visibility:public"],
+)
+
+internal_copied_filegroup(
+    name = "python_test_srcs",
+    srcs = glob(
+        [
+            "python/google/protobuf/internal/*_test.py",
+            "python/google/protobuf/internal/test_util.py",
+        ],
+    ),
+    include = "python",
+)
+
+py_proto_library(
+    name = "python_common_test_protos",
+    srcs = LITE_TEST_PROTOS + TEST_PROTOS,
+    include = "src",
+    protoc = ":protoc",
+    deps = [":python_proto"],
+)
+
+py_proto_library(
+    name = "python_specific_test_protos",
+    srcs = glob(["python/google/protobuf/internal/*.proto"]),
+    include = "python",
+    protoc = ":protoc",
+    deps = [":python_common_test_protos"],
+)
+
+py_library(
+    name = "python_tests",
+    srcs = [":python_test_srcs"],
+    deps = [
+        ":python_common_test_protos",
+        ":python_proto",
+        ":python_specific_test_protos",
+    ],
+)
+
+internal_protobuf_py_tests(
+    name = "python_tests_batch",
+    modules = [
+        "descriptor_database_test",
+        "descriptor_pool_test",
+        "descriptor_test",
+        "generator_test",
+        "json_format_test",
+        "message_factory_test",
+        # "message_test",      # failed due to testdata path
+        "proto_builder_test",
+        # "reflection_test",   # failed due to testdata path
+        "service_reflection_test",
+        "symbol_database_test",
+        "text_encoding_test",
+        # "text_format_test",  # failed due to testdata path
+        "unknown_fields_test",
+        "wire_format_test",
+    ],
+    deps = [":python_tests"],
+)
diff --git a/protobuf.bzl b/protobuf.bzl
index b83f7f5a..2199caf1 100644
--- a/protobuf.bzl
+++ b/protobuf.bzl
@@ -1,28 +1,54 @@
 # -*- mode: python; -*- PYTHON-PREPROCESSING-REQUIRED
 
-def _gen_dir(ctx):
-  if ctx.attr.include == None:
+def _GenDir(ctx):
+  if not ctx.attr.includes:
     return ""
-  if not ctx.attr.include:
+  if not ctx.attr.includes[0]:
     return ctx.label.package
   if not ctx.label.package:
-    return ctx.attr.include
-  return ctx.label.package + '/' + ctx.attr.include
+    return ctx.attr.includes[0]
+  return ctx.label.package + '/' + ctx.attr.includes[0]
 
-def _cc_outs(srcs):
+def _CcOuts(srcs):
   return [s[:-len(".proto")] +  ".pb.h" for s in srcs] + \
          [s[:-len(".proto")] + ".pb.cc" for s in srcs]
 
-def _py_outs(srcs):
+def _PyOuts(srcs):
   return [s[:-len(".proto")] + "_pb2.py" for s in srcs]
 
+def _RelativeOutputPath(path, include):
+  if include == None:
+    return path
+
+  if not path.startswith(include):
+    fail("Include path %s isn't part of the path %s." % (include, path))
+
+  if include and include[-1] != '/':
+    include = include + '/'
+
+  path = path[len(include):]
+
+  if not path.startswith(PACKAGE_NAME):
+    fail("The package %s is not within the path %s" % (PACKAGE_NAME, path))
+
+  if not PACKAGE_NAME:
+    return path
+
+  return path[len(PACKAGE_NAME)+1:]
+
+
+
 def _proto_gen_impl(ctx):
   """General implementation for generating protos"""
   srcs = ctx.files.srcs
   deps = []
   deps += ctx.files.srcs
-  gen_dir = _gen_dir(ctx)
-  import_flags = ["-I" + gen_dir]
+  gen_dir = _GenDir(ctx)
+  if gen_dir:
+    import_flags = ["-I" + gen_dir]
+  else:
+    import_flags = ["-I."]
+
   for dep in ctx.attr.deps:
     import_flags += dep.proto.import_flags
     deps += dep.proto.deps
@@ -53,7 +79,7 @@ _proto_gen = rule(
     attrs = {
         "srcs": attr.label_list(allow_files = True),
         "deps": attr.label_list(providers = ["proto"]),
-        "include": attr.string(),
+        "includes": attr.string_list(),
         "protoc": attr.label(
             executable = True,
             single_file = True,
@@ -73,7 +99,7 @@ def cc_proto_library(
         deps=[],
         cc_libs=[],
         include=None,
-        protoc=":protoc",
+        protoc="//google/protobuf:protoc",
         internal_bootstrap_hack=False,
         **kargs):
   """Bazel rule to create a C++ protobuf library from proto source files
@@ -94,6 +120,10 @@ def cc_proto_library(
 
   """
 
+  includes = []
+  if include != None:
+    includes = [include]
+
   if internal_bootstrap_hack:
     # For pre-checked-in generated files, we add the internal_bootstrap_hack
     # which will skip the codegen action.
@@ -101,7 +131,7 @@ def cc_proto_library(
         name=name + "_genproto",
         srcs=srcs,
         deps=[s + "_genproto" for s in deps],
-        include=include,
+        includes=includes,
         protoc=protoc,
     )
     # An empty cc_library to make rule dependency consistent.
@@ -110,20 +140,17 @@ def cc_proto_library(
         **kargs)
     return
 
-  outs = _cc_outs(srcs)
+  outs = _CcOuts(srcs)
   _proto_gen(
       name=name + "_genproto",
       srcs=srcs,
       deps=[s + "_genproto" for s in deps],
-      include=include,
+      includes=includes,
       protoc=protoc,
       gen_cc=1,
       outs=outs,
   )
 
-  includes = []
-  if include != None:
-    includes = [include]
 
   native.cc_library(
       name=name,
@@ -131,3 +158,111 @@ def cc_proto_library(
       deps=cc_libs + deps,
       includes=includes,
       **kargs)
+
+
+def internal_copied_filegroup(
+        name,
+        srcs,
+        include,
+        **kargs):
+  """Bazel rule to fix sources file to workaround with python path issues.
+
+  Args:
+    name: the name of the internal_copied_filegroup rule, which will be the
+        name of the generated filegroup.
+    srcs: the source files to be copied.
+    include: the expected import root of the source.
+    **kargs: extra arguments that will be passed into the filegroup.
+  """
+  outs = [_RelativeOutputPath(s, include) for s in srcs]
+
+  native.genrule(
+      name=name+"_genrule",
+      srcs=srcs,
+      outs=outs,
+      cmd=";".join(["cp $(location %s) $(location %s)" % \
+                    (s, _RelativeOutputPath(s, include)) \
+                    for s in srcs]))
+
+  native.filegroup(
+      name=name,
+      srcs=outs,
+      **kargs)
+
+
+def py_proto_library(
+        name,
+        srcs=[],
+        deps=[],
+        py_libs=[],
+        py_extra_srcs=[],
+        include=None,
+        protoc="//google/protobuf:protoc",
+        **kargs):
+  """Bazel rule to create a Python protobuf library from proto source files
+
+  Args:
+    name: the name of the py_proto_library.
+    srcs: the .proto files of the py_proto_library.
+    deps: a list of dependency labels; must be py_proto_library.
+    py_libs: a list of other py_library targets depended by the generated
+        py_library.
+    py_extra_srcs: extra source files that will be added to the output
+        py_library. This attribute is used for internal bootstrapping.
+    include: a string indicating the include path of the .proto files.
+    protoc: the label of the protocol compiler to generate the sources.
+    **kargs: other keyword arguments that are passed to cc_library.
+
+  """
+  outs = _PyOuts(srcs)
+
+  includes = []
+  if include != None:
+    includes = [include]
+
+  _proto_gen(
+      name=name + "_genproto",
+      srcs=srcs,
+      deps=[s + "_genproto" for s in deps],
+      includes=includes,
+      protoc=protoc,
+      gen_py=1,
+      outs=outs,
+  )
+
+  if include != None:
+    # Copy the output files to the desired location to make the import work.
+    internal_copied_filegroup_name=name + "_internal_copied_filegroup"
+    internal_copied_filegroup(
+        name=internal_copied_filegroup_name,
+        srcs=outs,
+        include=include)
+    outs=[internal_copied_filegroup_name]
+
+  native.py_library(
+      name=name,
+      srcs=outs+py_extra_srcs,
+      deps=py_libs+deps,
+      **kargs)
+
+def internal_protobuf_py_tests(
+    name,
+    modules=[],
+    **kargs):
+  """Bazel rules to create batch tests for protobuf internal.
+
+  Args:
+    name: the name of the rule.
+    modules: a list of modules for tests. The macro will create a py_test for
+        each of the parameter with the source "google/protobuf/%s.py"
+    kargs: extra parameters that will be passed into the py_test.
+
+  """
+  for m in modules:
+    s = _RelativeOutputPath(
+        "python/google/protobuf/internal/%s.py" % m, "python")
+    native.py_test(
+        name="py_%s" % m,
+        srcs=[s],
+        main=s,
+        **kargs)