From bb3edd22b36fb7aaea53666c8b010c2a9e316c80 Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Wed, 14 Feb 2024 16:41:36 -0800 Subject: [PATCH] [Python Modernization] Deprecate pkg_resources (#35849) Discuss thread about this change: [link](https://mail.google.com/mail/u/0/#sent/QgrcJHsBpWNGRlrMktwbppGGfFTVCFLcQgL?compose=new) Closes #35849 PiperOrigin-RevId: 607144827 --- setup.py | 17 ------------ src/python/grpcio/README.rst | 26 ------------------- src/python/grpcio_observability/setup.py | 17 ------------ .../protoc_plugin/_split_definitions_test.py | 25 ++++++++++++++++-- tools/distrib/python/grpcio_tools/README.rst | 26 ------------------- .../python/grpcio_tools/grpc_tools/command.py | 26 ++++++++++++++++--- tools/distrib/python/grpcio_tools/setup.py | 17 ------------ tools/distrib/python/xds_protos/build.py | 25 ++++++++++++++++-- 8 files changed, 68 insertions(+), 111 deletions(-) diff --git a/setup.py b/setup.py index 6ebfae1811e..2ce5fef4223 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,6 @@ import sys import sysconfig import _metadata -import pkg_resources from setuptools import Extension from setuptools.command import egg_info @@ -454,22 +453,6 @@ if "linux" in sys.platform or "darwin" in sys.platform: DEFINE_MACROS += (("PyMODINIT_FUNC", pymodinit),) DEFINE_MACROS += (("GRPC_POSIX_FORK_ALLOW_PTHREAD_ATFORK", 1),) -# By default, Python3 distutils enforces compatibility of -# c plugins (.so files) with the OSX version Python was built with. -# We need OSX 10.10, the oldest which supports C++ thread_local. -# Python 3.9: Mac OS Big Sur sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') returns int (11) -if "darwin" in sys.platform: - mac_target = sysconfig.get_config_var("MACOSX_DEPLOYMENT_TARGET") - if mac_target: - mac_target = pkg_resources.parse_version(str(mac_target)) - if mac_target < pkg_resources.parse_version("10.10.0"): - os.environ["MACOSX_DEPLOYMENT_TARGET"] = "10.10" - os.environ["_PYTHON_HOST_PLATFORM"] = re.sub( - r"macosx-[0-9]+\.[0-9]+-(.+)", - r"macosx-10.10-\1", - sysconfig.get_platform(), - ) - def cython_extensions_and_necessity(): cython_module_files = [ diff --git a/src/python/grpcio/README.rst b/src/python/grpcio/README.rst index 8e1826b0192..f3563e2e803 100644 --- a/src/python/grpcio/README.rst +++ b/src/python/grpcio/README.rst @@ -72,32 +72,6 @@ Troubleshooting Help, I ... -* **... see a** :code:`pkg_resources.VersionConflict` **when I try to install - grpc** - - This is likely because :code:`pip` doesn't own the offending dependency, - which in turn is likely because your operating system's package manager owns - it. You'll need to force the installation of the dependency: - - :code:`pip install --ignore-installed $OFFENDING_DEPENDENCY` - - For example, if you get an error like the following: - - :: - - Traceback (most recent call last): - File "", line 17, in - ... - File "/usr/lib/python2.7/dist-packages/pkg_resources.py", line 509, in find - raise VersionConflict(dist, req) - pkg_resources.VersionConflict: (six 1.8.0 (/usr/lib/python2.7/dist-packages), Requirement.parse('six>=1.10')) - - You can fix it by doing: - - :: - - sudo pip install --ignore-installed six - * **... see the following error on some platforms** :: diff --git a/src/python/grpcio_observability/setup.py b/src/python/grpcio_observability/setup.py index 4d4ebbeec57..f9bd1966504 100644 --- a/src/python/grpcio_observability/setup.py +++ b/src/python/grpcio_observability/setup.py @@ -22,7 +22,6 @@ from subprocess import PIPE import sys import sysconfig -import pkg_resources import setuptools from setuptools import Extension from setuptools.command import build_ext @@ -206,22 +205,6 @@ if "linux" in sys.platform or "darwin" in sys.platform: pymodinit = 'extern "C" __attribute__((visibility ("default"))) PyObject*' DEFINE_MACROS += (("PyMODINIT_FUNC", pymodinit),) -# By default, Python3 distutils enforces compatibility of -# c plugins (.so files) with the OSX version Python was built with. -# We need OSX 10.10, the oldest which supports C++ thread_local. -if "darwin" in sys.platform: - mac_target = sysconfig.get_config_var("MACOSX_DEPLOYMENT_TARGET") - if mac_target and ( - pkg_resources.parse_version(mac_target) - < pkg_resources.parse_version("10.10.0") - ): - os.environ["MACOSX_DEPLOYMENT_TARGET"] = "10.10" - os.environ["_PYTHON_HOST_PLATFORM"] = re.sub( - r"macosx-[0-9]+\.[0-9]+-(.+)", - r"macosx-10.10-\1", - sysconfig.get_platform(), - ) - def extension_modules(): if BUILD_WITH_CYTHON: diff --git a/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py b/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py index 9b8c3acf948..17e752adb42 100644 --- a/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py +++ b/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py @@ -26,7 +26,12 @@ import unittest import grpc from grpc_tools import protoc -import pkg_resources + +if sys.version_info >= (3, 9, 0): + from importlib import resources +else: + import pkg_resources + from tests.unit import test_common @@ -48,6 +53,22 @@ def _system_path(path_insertion): sys.path = old_system_path +def _get_resource_file_name( + package_or_requirement: str, resource_name: str +) -> str: + """Obtain the filename for a resource on the file system.""" + file_name = None + if sys.version_info >= (3, 9, 0): + file_name = ( + resources.files(package_or_requirement) / resource_name + ).resolve() + else: + file_name = pkg_resources.resource_filename( + package_or_requirement, resource_name + ) + return str(file_name) + + # NOTE(nathaniel): https://twitter.com/exoplaneteer/status/677259364256747520 # Life lesson "just always default to idempotence" reinforced. def _create_directory_tree(root, path_components_sequence): @@ -367,7 +388,7 @@ class WellKnownTypesTest(unittest.TestCase): def testWellKnownTypes(self): os.chdir(_TEST_DIR) out_dir = tempfile.mkdtemp(suffix="wkt_test", dir=".") - well_known_protos_include = pkg_resources.resource_filename( + well_known_protos_include = _get_resource_file_name( "grpc_tools", "_proto" ) args = [ diff --git a/tools/distrib/python/grpcio_tools/README.rst b/tools/distrib/python/grpcio_tools/README.rst index f0b5240f73c..05db96e33d5 100644 --- a/tools/distrib/python/grpcio_tools/README.rst +++ b/tools/distrib/python/grpcio_tools/README.rst @@ -75,32 +75,6 @@ Troubleshooting Help, I ... -* **... see a** :code:`pkg_resources.VersionConflict` **when I try to install - grpc** - - This is likely because :code:`pip` doesn't own the offending dependency, - which in turn is likely because your operating system's package manager owns - it. You'll need to force the installation of the dependency: - - :code:`pip install --ignore-installed $OFFENDING_DEPENDENCY` - - For example, if you get an error like the following: - - :: - - Traceback (most recent call last): - File "", line 17, in - ... - File "/usr/lib/python2.7/dist-packages/pkg_resources.py", line 509, in find - raise VersionConflict(dist, req) - pkg_resources.VersionConflict: (six 1.8.0 (/usr/lib/python2.7/dist-packages), Requirement.parse('six>=1.10')) - - You can fix it by doing: - - :: - - sudo pip install --ignore-installed six - * **... see compiler errors on some platforms when either installing from source or from the source distribution** If you see diff --git a/tools/distrib/python/grpcio_tools/grpc_tools/command.py b/tools/distrib/python/grpcio_tools/grpc_tools/command.py index 5b0b7eb6a79..33e43ffc6fb 100644 --- a/tools/distrib/python/grpcio_tools/grpc_tools/command.py +++ b/tools/distrib/python/grpcio_tools/grpc_tools/command.py @@ -16,9 +16,29 @@ import os import sys from grpc_tools import protoc -import pkg_resources import setuptools +if sys.version_info >= (3, 9, 0): + from importlib import resources +else: + import pkg_resources + + +def _get_resource_file_name( + package_or_requirement: str, resource_name: str +) -> str: + """Obtain the filename for a resource on the file system.""" + file_name = None + if sys.version_info >= (3, 9, 0): + file_name = ( + resources.files(package_or_requirement) / resource_name + ).resolve() + else: + file_name = pkg_resources.resource_filename( + package_or_requirement, resource_name + ) + return str(file_name) + def build_package_protos(package_root, strict_mode=False): proto_files = [] @@ -30,9 +50,7 @@ def build_package_protos(package_root, strict_mode=False): os.path.abspath(os.path.join(root, filename)) ) - well_known_protos_include = pkg_resources.resource_filename( - "grpc_tools", "_proto" - ) + well_known_protos_include = _get_resource_file_name("grpc_tools", "_proto") for proto_file in proto_files: command = [ diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index b479143387d..ee026efd00c 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -24,7 +24,6 @@ from subprocess import PIPE import sys import sysconfig -import pkg_resources import setuptools from setuptools import Extension from setuptools.command import build_ext @@ -214,22 +213,6 @@ if "win32" in sys.platform: elif "linux" in sys.platform or "darwin" in sys.platform: DEFINE_MACROS += (("HAVE_PTHREAD", 1),) -# By default, Python3 setuptools(distutils) enforces compatibility of -# c plugins (.so files) with the OSX version Python was built with. -# We need OSX 10.10, the oldest which supports C++ thread_local. -if "darwin" in sys.platform: - mac_target = sysconfig.get_config_var("MACOSX_DEPLOYMENT_TARGET") - if mac_target and ( - pkg_resources.parse_version(mac_target) - < pkg_resources.parse_version("10.10.0") - ): - os.environ["MACOSX_DEPLOYMENT_TARGET"] = "10.10" - os.environ["_PYTHON_HOST_PLATFORM"] = re.sub( - r"macosx-[0-9]+\.[0-9]+-(.+)", - r"macosx-10.10-\1", - sysconfig.get_platform(), - ) - def package_data(): tools_path = GRPC_PYTHON_TOOLS_PACKAGE.replace(".", os.path.sep) diff --git a/tools/distrib/python/xds_protos/build.py b/tools/distrib/python/xds_protos/build.py index cb5cc9bb1b7..9c1d2e9061e 100644 --- a/tools/distrib/python/xds_protos/build.py +++ b/tools/distrib/python/xds_protos/build.py @@ -15,15 +15,36 @@ """Builds the content of xds-protos package""" import os +import sys from grpc_tools import protoc -import pkg_resources + +if sys.version_info >= (3, 9, 0): + from importlib import resources +else: + import pkg_resources def localize_path(p): return os.path.join(*p.split("/")) +def _get_resource_file_name( + package_or_requirement: str, resource_name: str +) -> str: + """Obtain the filename for a resource on the file system.""" + file_name = None + if sys.version_info >= (3, 9, 0): + file_name = ( + resources.files(package_or_requirement) / resource_name + ).resolve() + else: + file_name = pkg_resources.resource_filename( + package_or_requirement, resource_name + ) + return str(file_name) + + # We might not want to compile all the protos EXCLUDE_PROTO_PACKAGES_LIST = tuple( localize_path(p) @@ -46,7 +67,7 @@ OPENCENSUS_PROTO_ROOT = os.path.join( GRPC_ROOT, "third_party", "opencensus-proto", "src" ) OPENTELEMETRY_PROTO_ROOT = os.path.join(GRPC_ROOT, "third_party", "opentelemetry") -WELL_KNOWN_PROTOS_INCLUDE = pkg_resources.resource_filename("grpc_tools", "_proto") +WELL_KNOWN_PROTOS_INCLUDE = _get_resource_file_name("grpc_tools", "_proto") OUTPUT_PATH = WORK_DIR