Merge pull request #24478 from gnossen/runtime_protos_wkt

[gRPC Easy] Make Well-Known Types Available to Runtime Protos

examples/python/route_guide/run_codegen.py

@@ -1,5 +1,3 @@
-#!/bin/bash -x
-
 # Copyright 2015 gRPC authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,11 +11,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+"""Runs protoc with the gRPC plugin to generate messages and gRPC stubs."""
 
-# Runs protoc with the gRPC plugin to generate messages and gRPC stubs
+from grpc_tools import protoc
 
-python3 -m grpc_tools.protoc \
-  -I ../../protos \
-  --python_out=. \
-  --grpc_python_out=. \
-  ../../protos/route_guide.proto
+protoc.main((
+    '',
+    '-I../../protos',
+    '--python_out=.',
+    '--grpc_python_out=.',
+    '../../protos/route_guide.proto',
+))

src/python/grpcio_tests/tests/unit/BUILD.bazel

@@ -125,6 +125,7 @@ py2and3_test(
     srcs = ["_dynamic_stubs_test.py"],
     data = [
         "data/foo/bar.proto",
+        "data/foo/bar_with_wkt.proto",
     ],
     imports = ["../../"],
     main = "_dynamic_stubs_test.py",

src/python/grpcio_tests/tests/unit/_dynamic_stubs_test.py

@@ -21,6 +21,8 @@ import os
 import sys
 import unittest
 
+_DATA_DIR = os.path.join("tests", "unit", "data")
+
 
 @contextlib.contextmanager
 def _grpc_tools_unimportable():
@@ -53,6 +55,18 @@ def _collect_errors(fn):
     return _wrapped
 
 
+def _python3_check(fn):
+
+    @functools.wraps(fn)
+    def _wrapped():
+        if sys.version_info[0] == 3:
+            fn()
+        else:
+            _assert_unimplemented("Python 3")
+
+    return _wrapped
+
+
 def _run_in_subprocess(test_case):
     sys.path.insert(
         0, os.path.join(os.path.realpath(os.path.dirname(__file__)), ".."))
@@ -80,24 +94,30 @@ def _assert_unimplemented(msg_substr):
 
 
 @_collect_errors
+@_python3_check
 def _test_sunny_day():
-    if sys.version_info[0] == 3:
-        import grpc
-        protos, services = grpc.protos_and_services(
-            os.path.join("tests", "unit", "data", "foo", "bar.proto"))
-        assert protos.BarMessage is not None
-        assert services.BarStub is not None
-    else:
-        _assert_unimplemented("Python 3")
+    import grpc
+    protos, services = grpc.protos_and_services(
+        os.path.join(_DATA_DIR, "foo", "bar.proto"))
+    assert protos.BarMessage is not None
+    assert services.BarStub is not None
 
 
 @_collect_errors
+@_python3_check
+def _test_well_known_types():
+    import grpc
+    protos, services = grpc.protos_and_services(
+        os.path.join(_DATA_DIR, "foo", "bar_with_wkt.proto"))
+    assert protos.BarMessage is not None
+    assert services.BarStub is not None
+
+
+@_collect_errors
+@_python3_check
 def _test_grpc_tools_unimportable():
     with _grpc_tools_unimportable():
-        if sys.version_info[0] == 3:
-            _assert_unimplemented("grpcio-tools")
-        else:
-            _assert_unimplemented("Python 3")
+        _assert_unimplemented("grpcio-tools")
 
 
 # NOTE(rbellevi): multiprocessing.Process fails to pickle function objects
@@ -109,6 +129,9 @@ class DynamicStubTest(unittest.TestCase):
     def test_sunny_day(self):
         _run_in_subprocess(_test_sunny_day)
 
+    def test_well_known_types(self):
+        _run_in_subprocess(_test_well_known_types)
+
     def test_grpc_tools_unimportable(self):
         _run_in_subprocess(_test_grpc_tools_unimportable)
 

src/python/grpcio_tests/tests/unit/data/foo/bar_with_wkt.proto

@@ -0,0 +1,28 @@
+// Copyright 2020 The gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package tests.unit.data.foo.bar;
+
+import "google/protobuf/any.proto";
+
+message BarMessage {
+  string a = 1;
+  google.protobuf.Any b = 2;
+};
+
+service Bar {
+  rpc GetBar(BarMessage) returns (BarMessage);
+};

tools/distrib/python/grpcio_tools/BUILD.bazel

@@ -19,6 +19,7 @@ package(default_visibility = [
 ])
 
 load("//bazel:cython_library.bzl", "pyx_library")
+load("grpcio_tools.bzl", "internal_copied_filegroup")
 
 cc_library(
     name = "protoc_lib",
@@ -37,12 +38,20 @@ pyx_library(
     deps = [":protoc_lib"],
 )
 
+internal_copied_filegroup(
+    name = "well_known_protos",
+    srcs = ["@com_google_protobuf//:well_known_protos"],
+    dest = "grpc_tools/_proto/",
+    strip_prefix = "src/",
+)
+
 py_library(
     name = "grpc_tools",
     srcs = [
         "grpc_tools/__init__.py",
         "grpc_tools/protoc.py",
     ],
+    data = [":well_known_protos"],
     imports = ["."],
     srcs_version = "PY2AND3",
     deps = [

tools/distrib/python/grpcio_tools/grpc_tools/protoc.py

@@ -58,6 +58,8 @@ if sys.version_info >= (3, 5, 0):
                     ProtoFinder(_SERVICE_MODULE_SUFFIX,
                                 _protoc_compiler.get_services)
                 ])
+                sys.path.append(
+                    pkg_resources.resource_filename('grpc_tools', '_proto'))
                 _FINDERS_INSTALLED = True
 
     def _module_name_to_proto_file(suffix, module_name):

tools/distrib/python/grpcio_tools/grpcio_tools.bzl

@@ -0,0 +1,94 @@
+# Copyright 2020 The gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+def _generate_copied_files_impl(ctx):
+    srcs = ctx.attr.srcs[0]
+    strip_prefix = ctx.attr.strip_prefix
+    dest = ctx.attr.dest
+
+    outs = []
+    for f in srcs.files.to_list():
+        destination_path = f.path
+        if f.path.startswith("external"):
+            external_separator = f.path.find("/")
+            repository_separator = f.path.find("/", external_separator + 1)
+            destination_path = f.path[repository_separator + 1:]
+        if not destination_path.startswith(strip_prefix):
+            fail("File '{}' did not start with '{}'.".format(
+                destination_path,
+                strip_prefix,
+            ))
+        destination_path = dest + destination_path[len(strip_prefix):]
+        destination_dir = destination_path.rfind("/")
+        out_file = ctx.actions.declare_file(destination_path)
+        outs.append(out_file)
+        ctx.actions.run_shell(
+            inputs = [f],
+            outputs = [out_file],
+            command = "mkdir -p {0} && cp {1} {2}".format(
+                out_file.dirname,
+                f.path,
+                out_file.path,
+            ),
+        )
+
+    return [DefaultInfo(files = depset(direct = outs))]
+
+_generate_copied_files = rule(
+    attrs = {
+        "srcs": attr.label_list(
+            mandatory = True,
+            allow_empty = False,
+        ),
+        "strip_prefix": attr.string(
+            default = "",
+        ),
+        "dest": attr.string(
+            mandatory = True,
+        ),
+    },
+    implementation = _generate_copied_files_impl,
+)
+
+def internal_copied_filegroup(name, srcs, strip_prefix, dest):
+    """Copies a file group to the current package.
+
+    Useful for using an existing filegroup as a data dependency.
+
+    Args:
+      name: The name of the rule.
+      srcs: A single filegroup.
+      strip_prefix: An optional string to strip from the beginning
+        of the path of each file in the filegroup. Must end in a slash.
+      dest: The directory in which to put the files, relative to the
+        current package. Must end in a slash.
+    """
+    if len(srcs) != 1:
+        fail("srcs must be a single filegroup.")
+
+    if not dest.endswith("/"):
+        fail("dest must end with a '/' character.")
+
+    _symlink_target = name + "_symlink"
+    _generate_copied_files(
+        name = _symlink_target,
+        srcs = srcs,
+        strip_prefix = strip_prefix,
+        dest = dest,
+    )
+
+    native.filegroup(
+        name = name,
+        srcs = [":" + _symlink_target],
+    )