Tightly integrate the Python plugin with its protoc

This grossly simplifies the protoc invocation to:
python -m grpc.protoc.compiler --python_out=... --grpc_python_out=...
[...] --plugin=protoc-gen-python-grpc=grpc_python_protoc_plugin... [...]

tools/distrib/python/check_grpcio_tools.py

@@ -37,16 +37,9 @@ OUT_OF_DATE_MESSAGE = """file {} is out of date
 
 Have you called tools/distrib/python/make_grpcio_tools.py since upgrading protobuf?"""
 
-check_protoc_deps_file = cStringIO.StringIO()
 check_protoc_lib_deps_file = cStringIO.StringIO()
-make.write_deps(make.BAZEL_DEPS_PROTOC_QUERY, check_protoc_deps_file)
 make.write_deps(make.BAZEL_DEPS_PROTOC_LIB_QUERY, check_protoc_lib_deps_file)
 
-with open(make.GRPC_PYTHON_PROTOC_DEPS, 'r') as protoc_deps_file:
-  if protoc_deps_file.read() != check_protoc_deps_file.getvalue():
-    print(OUT_OF_DATE_MESSAGE.format(make.GRPC_PYTHON_PROTOC_DEPS))
-    raise SystemExit(1)
-
 with open(make.GRPC_PYTHON_PROTOC_LIB_DEPS, 'r') as protoc_lib_deps_file:
   if protoc_lib_deps_file.read() != check_protoc_lib_deps_file.getvalue():
     print(OUT_OF_DATE_MESSAGE.format(make.GRPC_PYTHON_PROTOC_LIB_DEPS))

tools/distrib/python/grpcio_tools/grpc/protoc/compiler.py

@@ -31,8 +31,8 @@
 
 import sys
 
-from grpc.protoc import protoc
+from grpc.protoc import protoc_compiler
 
 
 if __name__ == '__main__':
-  protoc.run_main(sys.argv)
+  protoc_compiler.run_main(sys.argv)

tools/distrib/python/grpcio_tools/grpc/protoc/grpc_python_protoc_plugin.py

@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2016, Google Inc.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#     * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-#     * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-import sys
-
-from grpc.protoc import protoc_plugin
-
-
-if __name__ == '__main__':
-  protoc_plugin.run_main(sys.argv)

tools/distrib/python/grpcio_tools/grpc/protoc/main.cc

@@ -0,0 +1,25 @@
+#include <google/protobuf/compiler/command_line_interface.h>
+#include <google/protobuf/compiler/python/python_generator.h>
+
+#include "src/compiler/python_generator.h"
+
+#include "grpc/protoc/main.h"
+
+int main(int argc, char* argv[]) {
+  google::protobuf::compiler::CommandLineInterface cli;
+  cli.AllowPlugins("protoc-");
+
+  // Proto2 Python
+  google::protobuf::compiler::python::Generator py_generator;
+  cli.RegisterGenerator("--python_out", &py_generator,
+                        "Generate Python source file.");
+
+  // gRPC Python
+  grpc_python_generator::GeneratorConfiguration grpc_py_config;
+  grpc_py_config.beta_package_root = "grpc.beta";
+  grpc_python_generator::PythonGrpcGenerator grpc_py_generator(grpc_py_config);
+  cli.RegisterGenerator("--grpc_python_out", &grpc_py_generator,
+                        "Generate Python source file.");
+
+  return cli.Run(argc, argv);
+}

tools/distrib/python/grpcio_tools/grpc/protoc/protoc_plugin.pyx

@@ -1,39 +0,0 @@
-# Copyright 2016, Google Inc.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#     * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-#     * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-from libc cimport stdlib
-
-cdef extern from "grpc/protoc/main.h":
-  int main(int argc, char *argv[])
-
-def run_main(list args not None):
-  cdef char **argv = <char **>stdlib.malloc(len(args)*sizeof(char *))
-  for i in range(len(args)):
-    argv[i] = args[i]
-  return main(len(args), argv)

tools/distrib/python/grpcio_tools/setup.py

@@ -40,32 +40,17 @@ from setuptools.command import build_ext
 os.chdir(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, os.path.abspath('.'))
 
-import protoc_deps
 import protoc_lib_deps
 
 def protoc_ext_module():
-  protoc_sources = [
-      os.path.join('third_party/protobuf/src', cc_file)
-      for cc_file in protoc_deps.CC_FILES]
-  protoc_ext = extension.Extension(
-    name='grpc.protoc.protoc',
-    sources=['grpc/protoc/protoc.pyx'] + protoc_sources,
-    include_dirs=['.', 'third_party/protobuf/src'],
-    language='c++',
-    define_macros=[('HAVE_PTHREAD', 1)],
-    extra_compile_args=['-lpthread', '-frtti'],
-  )
-  return protoc_ext
-
-def plugin_ext_module():
   plugin_sources = [
-      'grpc_root/src/compiler/python_generator.cc',
-      'grpc_root/src/compiler/python_plugin.cc'] + [
+      'grpc/protoc/main.cc',
+      'grpc_root/src/compiler/python_generator.cc'] + [
       os.path.join('third_party/protobuf/src', cc_file)
       for cc_file in protoc_lib_deps.CC_FILES]
   plugin_ext = extension.Extension(
-      name='grpc.protoc.protoc_plugin',
-      sources=['grpc/protoc/protoc_plugin.pyx'] + plugin_sources,
+      name='grpc.protoc.protoc_compiler',
+      sources=['grpc/protoc/protoc_compiler.pyx'] + plugin_sources,
       include_dirs=[
           '.',
           'grpc_root',
@@ -74,7 +59,7 @@ def plugin_ext_module():
       ],
       language='c++',
       define_macros=[('HAVE_PTHREAD', 1)],
-      extra_compile_args=['-lpthread', '-std=c++11'],
+      extra_compile_args=['-lpthread', '-frtti', '-std=c++11'],
   )
   return plugin_ext
 
@@ -88,12 +73,7 @@ setuptools.setup(
   license='',
   ext_modules=maybe_cythonize([
       protoc_ext_module(),
-      plugin_ext_module(),
   ]),
-  scripts=[
-    'grpc/protoc/grpc_python_protoc_compiler.py',
-    'grpc/protoc/grpc_python_protoc_plugin.py',
-  ],
   packages=setuptools.find_packages('.'),
   namespace_packages=['grpc'],
   install_requires=[

tools/distrib/python/make_grpcio_tools.py

@@ -64,7 +64,7 @@ DEPS_FILE_CONTENT="""
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-# AUTO-GENERATED BY mk_grpcio_tools.py!
+# AUTO-GENERATED BY make_grpcio_tools.py!
 CC_FILES={}
 """
 
@@ -83,8 +83,6 @@ GRPC_PYTHON_PROTOBUF = os.path.join(GRPC_PYTHON_ROOT,
                                     'third_party/protobuf/src')
 GRPC_PYTHON_PROTOC_PLUGINS = os.path.join(GRPC_PYTHON_ROOT,
                                           'grpc_root/src/compiler')
-GRPC_PYTHON_PROTOC_DEPS = os.path.join(GRPC_PYTHON_ROOT,
-                                       'protoc_deps.py')
 GRPC_PYTHON_PROTOC_LIB_DEPS = os.path.join(GRPC_PYTHON_ROOT,
                                            'protoc_lib_deps.py')
 
@@ -92,7 +90,6 @@ GRPC_INCLUDE = os.path.join(GRPC_ROOT, 'include')
 GRPC_PYTHON_INCLUDE = os.path.join(GRPC_PYTHON_ROOT, 'grpc_root/include')
 
 BAZEL_DEPS = os.path.join(GRPC_ROOT, 'tools/distrib/python/bazel_deps.sh')
-BAZEL_DEPS_PROTOC_QUERY = '//:protoc'
 BAZEL_DEPS_PROTOC_LIB_QUERY = '//:protoc_lib'
 
 
@@ -123,8 +120,6 @@ def main():
   shutil.copytree(GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS)
   shutil.copytree(GRPC_INCLUDE, GRPC_PYTHON_INCLUDE)
 
-  with open(GRPC_PYTHON_PROTOC_DEPS, 'w') as deps_file:
-    write_deps(BAZEL_DEPS_PROTOC_QUERY, deps_file)
   with open(GRPC_PYTHON_PROTOC_LIB_DEPS, 'w') as deps_file:
     write_deps(BAZEL_DEPS_PROTOC_LIB_QUERY, deps_file)