Get rid of distutils (#22281)

* Get rid of distutils

* get rid of LooseVersion

* try to get rid of clean and copy_file

* fix mo tests

* fix warning in mo

* Update src/bindings/python/wheel/setup.py

Co-authored-by: Jan Iwaszkiewicz <jan.iwaszkiewicz@intel.com>

* add packaging to requirements

---------

Co-authored-by: Jan Iwaszkiewicz <jan.iwaszkiewicz@intel.com>

src/bindings/python/requirements.txt

@@ -1,2 +1,3 @@
 numpy>=1.16.6
 openvino-telemetry>=2023.2.1
+packaging

src/bindings/python/src/openvino/frontend/tensorflow/utils.py

@@ -7,7 +7,7 @@
 
 import logging as log
 import sys
-from distutils.version import LooseVersion
+from packaging.version import parse, Version
 from typing import List, Dict, Union
 
 import numpy as np
@@ -369,7 +369,7 @@ def extract_model_graph(argv):
     if isinstance(model, tf.compat.v1.Session):
         argv["input_model"] = model.graph
         return True
-    if env_setup["tensorflow"] >= LooseVersion("2.6.0") and isinstance(model, (tf.types.experimental.GenericFunction,
+    if Version(env_setup["tensorflow"]) >= parse("2.6.0") and isinstance(model, (tf.types.experimental.GenericFunction,
                                                                                tf.types.experimental.ConcreteFunction)):
         return True
     if isinstance(model, tf.train.Checkpoint):

src/bindings/python/wheel/setup.py

@@ -9,19 +9,19 @@ import subprocess  # nosec
 import typing
 import platform
 import re
+import shutil
 import multiprocessing
+import logging as log
 from fnmatch import fnmatchcase
 from pathlib import Path
 from shutil import copyfile, rmtree
-from setuptools import setup, find_namespace_packages, Extension
+from setuptools import setup, find_namespace_packages, Extension, Command
 from setuptools.command.build_ext import build_ext
 from setuptools.command.build_clib import build_clib
 from setuptools.command.install import install
-from distutils.command.build import build
+from setuptools.command.build import build
-from distutils.command.clean import clean
+from setuptools.errors import SetupError
-from distutils.errors import DistutilsSetupError
+
-from distutils.file_util import copy_file
-from distutils import log
 
 WHEEL_LIBS_INSTALL_DIR = os.path.join("openvino", "libs")
 WHEEL_LIBS_PACKAGE = "openvino.libs"
@@ -197,7 +197,7 @@ class PrebuiltExtension(Extension):
     def __init__(self, name, sources, *args, **kwargs):
         if len(sources) != 1:
             nln = "\n"
-            raise DistutilsSetupError(f"PrebuiltExtension can accept only one source, but got: {nln}{nln.join(sources)}")
+            raise SetupError(f"PrebuiltExtension can accept only one source, but got: {nln}{nln.join(sources)}")
         super().__init__(name, sources, *args, **kwargs)
         self._needs_stub = False
 
@@ -419,6 +419,16 @@ class PrepareLibs(build_clib):
             package_data.update({WHEEL_LIBS_PACKAGE: ["*"]})
 
 
+def copy_file(src, dst, verbose=False, dry_run=False):
+    """Custom file copy."""
+    if dry_run:
+        log.info(f"DRY RUN: Would copy '{src}' to '{dst}'")
+    else:
+        shutil.copyfile(src, dst)
+        if verbose:
+            log.info(f"Copied '{src}' to '{dst}'")
+
+
 class CopyExt(build_ext):
     """Copy extension files to the build directory."""
     def run(self):
@@ -427,7 +437,7 @@ class CopyExt(build_ext):
 
         for extension in self.extensions:
             if not isinstance(extension, PrebuiltExtension):
-                raise DistutilsSetupError(f"build_ext can accept PrebuiltExtension only, but got {extension.name}")
+                raise SetupError(f"build_ext can accept PrebuiltExtension only, but got {extension.name}")
             src = extension.sources[0]
             dst = self.get_ext_fullpath(extension.name)
             os.makedirs(os.path.dirname(dst), exist_ok=True)
@@ -448,9 +458,17 @@ class CustomInstall(install):
         install.run(self)
 
 
-class CustomClean(clean):
+class CustomClean(Command):
     """Clean up staging directories."""
 
+    user_options = []
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
     def clean_install_prefix(self, install_cfg):
         for comp, comp_data in install_cfg.items():
             install_prefix = comp_data.get("prefix")
@@ -461,7 +479,6 @@ class CustomClean(clean):
     def run(self):
         self.clean_install_prefix(LIB_INSTALL_CFG)
         self.clean_install_prefix(PY_INSTALL_CFG)
-        clean.run(self)
 
 
 def ignore_patterns(*patterns):
@@ -509,7 +526,7 @@ def set_rpath(rpath, binary):
     if sys.platform == "linux":
         with open(os.path.realpath(binary), "rb") as file:
             if file.read(1) != b"\x7f":
-                log.warn(f"WARNING: {binary}: missed ELF header")
+                log.warning(f"WARNING: {binary}: missed ELF header")
                 return
         rpath_tool = "patchelf"
         cmd = [rpath_tool, "--set-rpath", rpath, binary, "--force-rpath"]

src/plugins/intel_cpu/tools/commit_slider/commit_slider.py

@@ -2,7 +2,7 @@ import subprocess
 import os
 import shutil
 import sys
-from distutils.dir_util import copy_tree
+from shutil import copytree
 from utils.helpers import safeClearDir, getParams
 
 args, cfgData, customCfgPath = getParams()
@@ -45,7 +45,7 @@ else:
     else:
         safeClearDir(workPath, cfgData)
     curPath = os.getcwd()
-    copy_tree(curPath, workPath)
+    copytree(curPath, workPath)
     scriptName = os.path.basename(__file__)
     argString = " ".join(sys.argv)
     formattedCmd = "{py} {workPath}/{argString} -wd".format(
@@ -57,12 +57,12 @@ else:
     tempLogPath = cfgData["logPath"].format(workPath=workPath)
     permLogPath = cfgData["logPath"].format(workPath=curPath)
     safeClearDir(permLogPath, cfgData)
-    copy_tree(tempLogPath, permLogPath)
+    copytree(tempLogPath, permLogPath)
 
     tempCachePath = cfgData["cachePath"].format(workPath=workPath)
     permCachePath = cfgData["cachePath"].format(workPath=curPath)
     safeClearDir(permCachePath, cfgData)
-    copy_tree(tempCachePath, permCachePath)
+    copytree(tempCachePath, permCachePath)
 
     shutil.copyfile(
         os.path.join(workPath, customCfgPath),

tests/layer_tests/requirements.txt

@@ -6,6 +6,7 @@ requests
 torch
 torchvision
 transformers
+packaging
 pytest
 tensorflow-addons; python_version <= '3.10'
 jax; sys_platform == "linux" and platform_machine == "x86_64" # https://jax.readthedocs.io/en/latest/installation.html#pip-installation-cpu - wheels are for "x86_64" only

tests/samples_tests/smoke_tests/common/samples_common_test_class.py

@@ -26,7 +26,7 @@ import zipfile
 
 import logging as log
 from common.common_utils import shell
-from distutils import spawn
+from shutil import which
 
 log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout)
 
@@ -168,7 +168,7 @@ class SamplesCommonTestClass():
 
         # This exeption is made for benchmark_app, because it locates in another place.
         if 'benchmark_app' in path2 and 'python' in sample_type.lower():
-            executable_path = spawn.find_executable(str('benchmark_app'))
+            executable_path = which(str('benchmark_app'))
         # if not hasattr(cls, 'executable_path'):
         cls.executable_path = executable_path
 

tests/stress_tests/scripts/get_testdata.py

@@ -15,7 +15,7 @@ import os
 import shutil
 import subprocess
 import sys
-from distutils.dir_util import copy_tree
+from shutil import copytree
 from inspect import getsourcefile
 from pathlib import Path
 import defusedxml.ElementTree as ET
@@ -232,7 +232,7 @@ def main():
     for ir_src_path in args.omz_models_out_dir.rglob("*.xml"):
         ir_dst_path = args.omz_irs_out_dir / os.path.relpath(ir_src_path, args.omz_models_out_dir)
         # allows copy to an existing folder
-        copy_tree(str(ir_src_path.parent), str(ir_dst_path.parent))
+        copytree(str(ir_src_path.parent), str(ir_dst_path.parent))
 
 
 if __name__ == "__main__":

tools/mo/.pylintrc

@@ -202,7 +202,7 @@ ignore-mixin-members=yes
 # (useful for modules/projects where namespaces are manipulated during runtime
 # and thus existing member attributes cannot be deduced by static analysis. It
 # supports qualified module names, as well as Unix pattern matching.
-ignored-modules=flask_sqlalchemy,app.extensions.flask_sqlalchemy,distutils,openvino,torch,paddle
+ignored-modules=flask_sqlalchemy,app.extensions.flask_sqlalchemy,openvino,torch,paddle
 
 # List of class names for which member attributes should not be checked (useful
 # for classes with dynamically set attributes). This supports the use of

tools/mo/openvino/tools/mo/front/tf/loader.py

@@ -5,7 +5,7 @@ import argparse
 import logging as log
 import os
 import re
-from distutils.version import LooseVersion
+from packaging.version import parse, Version
 from pathlib import Path
 
 from openvino.tools.mo.graph.graph import Node
@@ -175,7 +175,7 @@ def deducing_metagraph_path(meta_graph_file: str):
 
 def freeze_tf2_concrete_function(model, concrete_func, env_setup):
 
-    if "tensorflow" in env_setup and env_setup["tensorflow"] >= LooseVersion("2.2.0"):
+    if "tensorflow" in env_setup and Version(env_setup["tensorflow"]) >= parse("2.2.0"):
         frozen_func = convert_variables_to_constants_v2(concrete_func,
                                                         lower_control_flow=False,
                                                         aggressive_inlining=True)  # pylint: disable=E1123
@@ -226,7 +226,7 @@ def prepare_graph_def(model):
 
         conc_func = tf_function.get_concrete_function(model_inputs)
         return freeze_tf2_concrete_function(model, conc_func, env_setup)
-    if env_setup["tensorflow"] >= LooseVersion("2.6.0") and isinstance(model, tf.types.experimental.GenericFunction):
+    if Version(env_setup["tensorflow"]) >= parse("2.6.0") and isinstance(model, tf.types.experimental.GenericFunction):
 
         assert hasattr(model, "input_signature") and model.input_signature is not None, \
             "'input_signature' needs to be set for model conversion."

tools/mo/openvino/tools/mo/moc_frontend/pipeline.py

@@ -101,7 +101,7 @@ def moc_pipeline(argv: argparse.Namespace, moc_front_end: FrontEnd):
                 model.add_name_for_tensor(new_input['node'], new_input['input_name'])
             except NotImplementedFailure as e:
                 # some frontends might not implement this method
-                log.warn('Could not add an additional name to a tensor pointed to by \'{}\'. Details: {}'.format(
+                log.warning('Could not add an additional name to a tensor pointed to by \'{}\'. Details: {}'.format(
                     new_input['input_name'], str(e)))
 
     enabled_transforms, disabled_transforms = get_enabled_and_disabled_transforms()

tools/mo/openvino/tools/mo/utils/cli_parser.py

@@ -7,7 +7,6 @@ import logging as log
 import os
 import re
 from collections import OrderedDict, namedtuple
-from distutils.util import strtobool
 from itertools import zip_longest
 from pathlib import Path
 from operator import xor
@@ -25,6 +24,22 @@ from openvino.tools.mo.utils.utils import refer_to_faq_msg, get_mo_root_dir
 from openvino.tools.mo.utils.help import get_convert_model_help_specifics, get_to_string_methods_for_params
 
 
+def strtobool(value):
+    """
+    Converts a string representation to true or false.
+
+    :param value: a string which will be converted to bool.
+    :return: boolean value of the input string.
+    """
+    value = value.lower()
+    if value in ('y', 'yes', 't', 'true', 'on', '1'):
+        return 1
+    elif value in ('n', 'no', 'f', 'false', 'off', '0'):
+        return 0
+    else:
+        raise ValueError(f"Invalid truth value: {value}.")
+
+
 def extension_path_to_str_or_extensions_class(extension):
     if isinstance(extension, str):
         return extension

tools/mo/requirements.txt

@@ -4,3 +4,4 @@ importlib-metadata; python_version < "3.8" and sys_platform == "win32"
 networkx
 defusedxml
 openvino-telemetry
+packaging

tools/openvino_dev/setup.py

@@ -13,9 +13,9 @@ import platform
 import subprocess  # nosec
 import shutil
 import re
-from distutils import log
+import logging as log
-from distutils.command.build import build
+from setuptools import Command
-from distutils.command.clean import clean
+from setuptools.command.build import build
 from pathlib import Path
 from fnmatch import fnmatchcase
 import pkg_resources
@@ -153,9 +153,17 @@ class CustomInstall(install):
         install.run(self)
 
 
-class CustomClean(clean):
+class CustomClean(Command):
     """Clean up staging directories"""
 
+    user_options = []
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
     def clean_temp_files(self):
         """Clean components staging directories"""
         for pattern in './build ./dist **/*.pyc **/*.tgz **/*.egg-info'.split(' '):
@@ -173,7 +181,6 @@ class CustomClean(clean):
 
     def run(self):
         self.clean_temp_files()
-        clean.run(self)
 
 
 def get_description(desc_file_path):

tools/ovc/.pylintrc

@@ -202,7 +202,7 @@ ignore-mixin-members=yes
 # (useful for modules/projects where namespaces are manipulated during runtime
 # and thus existing member attributes cannot be deduced by static analysis. It
 # supports qualified module names, as well as Unix pattern matching.
-ignored-modules=flask_sqlalchemy,app.extensions.flask_sqlalchemy,distutils,openvino,torch,paddle
+ignored-modules=flask_sqlalchemy,app.extensions.flask_sqlalchemy,openvino,torch,paddle
 
 # List of class names for which member attributes should not be checked (useful
 # for classes with dynamically set attributes). This supports the use of

tools/ovc/openvino/tools/ovc/moc_frontend/pipeline.py

@@ -104,7 +104,7 @@ def moc_pipeline(argv: argparse.Namespace, moc_front_end: FrontEnd):
                 model.add_name_for_tensor(new_input['node'], new_input['input_name'])
             except NotImplementedFailure as e:
                 # some frontends might not implement this method
-                log.warn('Could not add an additional name to a tensor pointed to by \'{}\'. Details: {}'.format(
+                log.warning('Could not add an additional name to a tensor pointed to by \'{}\'. Details: {}'.format(
                     new_input['input_name'], str(e)))
 
     enabled_transforms, disabled_transforms = get_enabled_and_disabled_transforms()