From b1b1014a34b6658b181ec50d4fa23f36883f524f Mon Sep 17 00:00:00 2001 From: Ilya Lavrenov Date: Thu, 8 Jun 2023 15:35:27 +0400 Subject: [PATCH] Enable python API in deb / rpm packages (#17923) * Enable python API in debian package * Enable python API in debian package --- .ci/azure/linux_debian.yml | 109 +++++++++--------- .../packaging/debian/debian.cmake | 20 +++- .../developer_package/packaging/rpm/rpm.cmake | 18 ++- cmake/features.cmake | 4 + cmake/packaging/debian.cmake | 47 ++++---- cmake/packaging/rpm.cmake | 58 +++++++--- 6 files changed, 154 insertions(+), 102 deletions(-) diff --git a/.ci/azure/linux_debian.yml b/.ci/azure/linux_debian.yml index 636e1cc6617..a3be74fbb97 100644 --- a/.ci/azure/linux_debian.yml +++ b/.ci/azure/linux_debian.yml @@ -62,6 +62,8 @@ jobs: SAMPLES_INSTALL_DIR: /usr/share/openvino/samples PYTHON_SAMPLES_INSTALL_DIR: $(INSTALL_DIR)/share/openvino/samples/python PYTHON_WHEEL_INSTALL_DIR: $HOME/.local/lib/python3.8/site-packages + BUILD_VENV: $(WORK_DIR)/build_venv + TEST_VENV: $(WORK_DIR)/test_venv TMP_DIR: /mnt/tmp SHARE_DIR: /mount/cinfsshare/onnxtestdata CCACHE_DIR: $(SHARE_DIR)/ccache/master/linux @@ -111,25 +113,32 @@ jobs: # 'clang' is used as a default compiler sudo apt --assume-yes install clang sudo apt --assume-yes install --no-install-recommends libopencv-imgproc-dev libopencv-imgcodecs-dev - # For opencv-python: python3-setuptools and pip upgrade - python3 -m pip install --upgrade pip - python3 -m pip install -r $(REPO_DIR)/src/bindings/python/wheel/requirements-dev.txt - python3 -m pip install -r $(REPO_DIR)/src/bindings/python/requirements.txt + # install build dependencies + (cd $(WORK_DIR) && python3 -m venv build_venv) + $(BUILD_VENV)/bin/python3 -m pip install -U pip + $(BUILD_VENV)/bin/python3 -m pip install -r $(REPO_DIR)/src/bindings/python/wheel/requirements-dev.txt + $(BUILD_VENV)/bin/python3 -m pip install -r $(REPO_DIR)/src/bindings/python/requirements.txt # For running Python API tests - python3 -m pip install -r $(REPO_DIR)/src/bindings/python/src/compatibility/openvino/requirements-dev.txt + $(BUILD_VENV)/bin/python3 -m pip install -r $(REPO_DIR)/src/bindings/python/src/compatibility/openvino/requirements-dev.txt # For running Paddle frontend unit tests - python3 -m pip install -r $(REPO_DIR)/src/frontends/paddle/tests/requirements.txt + $(BUILD_VENV)/bin/python3 -m pip install -r $(REPO_DIR)/src/frontends/paddle/tests/requirements.txt # For running ONNX frontend unit tests - python3 -m pip install -r $(REPO_DIR)/src/frontends/onnx/tests/requirements.txt + $(BUILD_VENV)/bin/python3 -m pip install -r $(REPO_DIR)/src/frontends/onnx/tests/requirements.txt # For running TensorFlow frontend unit tests - python3 -m pip install -r $(REPO_DIR)/src/frontends/tensorflow/tests/requirements.txt + $(BUILD_VENV)/bin/python3 -m pip install -r $(REPO_DIR)/src/frontends/tensorflow/tests/requirements.txt # For MO unit tests - python3 -m pip install -r $(REPO_DIR)/tools/mo/requirements_mxnet.txt - python3 -m pip install -r $(REPO_DIR)/tools/mo/requirements_caffe.txt - python3 -m pip install -r $(REPO_DIR)/tools/mo/requirements_kaldi.txt - python3 -m pip install -r $(REPO_DIR)/tools/mo/requirements_onnx.txt - python3 -m pip install -r $(REPO_DIR)/tools/mo/requirements_tf2.txt - python3 -m pip install -r $(REPO_DIR)/tools/mo/requirements_dev.txt + (cd $(WORK_DIR) && python3 -m venv test_venv) + $(TEST_VENV)/bin/python3 -m pip install -U pip + $(TEST_VENV)/bin/python3 -m pip install -r $(REPO_DIR)/tools/mo/requirements_mxnet.txt + $(TEST_VENV)/bin/python3 -m pip install -r $(REPO_DIR)/tools/mo/requirements_caffe.txt + $(TEST_VENV)/bin/python3 -m pip install -r $(REPO_DIR)/tools/mo/requirements_kaldi.txt + $(TEST_VENV)/bin/python3 -m pip install -r $(REPO_DIR)/tools/mo/requirements_onnx.txt + $(TEST_VENV)/bin/python3 -m pip install -r $(REPO_DIR)/tools/mo/requirements_tf2.txt + $(TEST_VENV)/bin/python3 -m pip install -r $(REPO_DIR)/tools/mo/requirements_dev.txt + $(TEST_VENV)/bin/python3 -m pip install -r $(REPO_DIR)/src/frontends/paddle/tests/requirements.txt + # for Python API tests + /usr/bin/python3 -m pip install -r $(REPO_DIR)/src/bindings/python/requirements_test.txt + /usr/bin/python3 -m pip uninstall -y numpy # apt-get install python3-numpy will be used # Speed up build sudo apt -y --no-install-recommends install unzip wget https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-linux.zip @@ -137,7 +146,7 @@ jobs: sudo cp -v ninja /usr/local/bin/ # Speed up tests git clone https://github.com/google/gtest-parallel.git - displayName: 'Install dependencies' + displayName: 'Install build dependencies' # Should be after 'Install dependencies' because Git lfs is not installed - checkout: testdata @@ -155,7 +164,7 @@ jobs: -DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DENABLE_PYTHON=ON -DENABLE_INTEL_GNA=OFF - -DPYTHON_EXECUTABLE=/usr/bin/python3.8 + -DPYTHON_EXECUTABLE=$(BUILD_VENV)/bin/python3 -DENABLE_TESTS=ON -DENABLE_FASTER_BUILD=ON -DENABLE_STRICT_DEPENDENCIES=OFF @@ -164,6 +173,7 @@ jobs: -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_LINKER_LAUNCHER=ccache -DCMAKE_C_LINKER_LAUNCHER=ccache + -DENABLE_PYTHON_PACKAGING=ON -DCPACK_GENERATOR=DEB -S $(REPO_DIR) -B $(BUILD_DIR) @@ -213,9 +223,6 @@ jobs: - script: cmake -DCOMPONENT=tests -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) -P $(BUILD_LAYER_TESTS_DIR)/cmake_install.cmake displayName: 'Install Layer Tests' - - script: python3 -m pip install openvino-dev --find-links=$(INSTALL_DIR)/tools - displayName: 'Install python wheels' - - script: cmake -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) -DCOMPONENT=tests -P $(BUILD_DIR)/cmake_install.cmake displayName: 'Install tests' @@ -241,7 +248,7 @@ jobs: rm -r _CPack_Packages dpkg-scanpackages . /dev/null | gzip -9c > Packages.gz echo "deb [trusted=yes] file:$(BUILD_DIR) ./" | sudo tee /etc/apt/sources.list.d/openvino-local.list - sudo apt-get update -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/openvino-local.list + sudo apt-get update sudo apt-get install openvino -y workingDirectory: $(BUILD_DIR) displayName: 'Install Debian packages' @@ -264,14 +271,12 @@ jobs: - script: $(SAMPLES_INSTALL_DIR)/c/build_samples.sh -i $(INSTALL_DIR) displayName: 'Build c samples' - - script: | - $(INSTALL_TEST_DIR)/ov_core_unit_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU* --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-NGraphUT.xml + - script: $(INSTALL_TEST_DIR)/ov_core_unit_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU* --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-NGraphUT.xml env: LD_LIBRARY_PATH: $(INSTALL_TEST_DIR) displayName: 'OV Core UT' - - script: | - $(INSTALL_TEST_DIR)/ov_onnx_frontend_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU* --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-ONNXFrontend.xml + - script: $(INSTALL_TEST_DIR)/ov_onnx_frontend_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU* --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-ONNXFrontend.xml env: LD_LIBRARY_PATH: $(INSTALL_TEST_DIR) displayName: 'ONNX Frontend Tests' @@ -288,14 +293,12 @@ jobs: LD_LIBRARY_PATH: $(INSTALL_TEST_DIR) displayName: 'TensorFlow Frontend Unit Tests' - - script: | - $(INSTALL_TEST_DIR)/ov_tensorflow_common_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-TensorflowCommon.xml + - script: $(INSTALL_TEST_DIR)/ov_tensorflow_common_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-TensorflowCommon.xml env: LD_LIBRARY_PATH: $(INSTALL_TEST_DIR) displayName: 'TensorFlow Common Unit Tests' - - script: | - $(INSTALL_TEST_DIR)/ov_tensorflow_lite_frontend_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-TensorflowLite.xml + - script: $(INSTALL_TEST_DIR)/ov_tensorflow_lite_frontend_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-TensorflowLite.xml env: LD_LIBRARY_PATH: $(INSTALL_TEST_DIR) displayName: 'TensorFlow Lite Frontend Unit Tests' @@ -306,14 +309,12 @@ jobs: - script: $(INSTALL_TEST_DIR)/ieMultiPluginUnitTests --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-ieMultiPluginUnitTests.xml displayName: 'MULTI UT' - - script: | - $(INSTALL_TEST_DIR)/ov_template_func_tests --gtest_filter=*smoke* --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-templateFuncTests.xml + - script: $(INSTALL_TEST_DIR)/ov_template_func_tests --gtest_filter=*smoke* --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-templateFuncTests.xml env: LD_LIBRARY_PATH: $(INSTALL_TEST_DIR) displayName: 'TEMPLATE FuncTests' - - script: | - $(INSTALL_TEST_DIR)/InferenceEngineCAPITests --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-InferenceEngineCAPITests.xml + - script: $(INSTALL_TEST_DIR)/InferenceEngineCAPITests --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-InferenceEngineCAPITests.xml env: DATA_PATH: $(MODELS_PATH) MODELS_PATH: $(MODELS_PATH) @@ -327,7 +328,7 @@ jobs: # Skip test_onnx/test_zoo_models and test_onnx/test_backend due to long execution time - script: | - python3 -m pytest -s $(INSTALL_TEST_DIR)/pyngraph \ + /usr/bin/python3 -m pytest -s $(INSTALL_TEST_DIR)/pyngraph \ --junitxml=$(INSTALL_TEST_DIR)/TEST-Pyngraph.xml \ --ignore=$(INSTALL_TEST_DIR)/pyngraph/tests/test_onnx/test_zoo_models.py \ --ignore=$(INSTALL_TEST_DIR)/pyngraph/tests/test_onnx/test_backend.py @@ -337,22 +338,23 @@ jobs: # Skip test_onnx/test_zoo_models and test_onnx/test_backend due to long execution time - script: | - # Required by python imports to load requires libraries - # - tests install dir for mock_py - # - OpenVINO wheel installation dir for others frontend required by mock_py (is not part of wheel pkg) - export LD_LIBRARY_PATH=$(PYTHON_WHEEL_INSTALL_DIR)/openvino/libs:$(INSTALL_TEST_DIR):$LD_LIBRARY_PATH - # For python imports to import pybind_mock_frontend - export PYTHONPATH=$(INSTALL_TEST_DIR):$PYTHONPATH - python3 -m pytest -s $(INSTALL_TEST_DIR)/pyopenvino \ + /usr/bin/python3 -m pytest -s $(INSTALL_TEST_DIR)/pyopenvino \ --junitxml=$(INSTALL_TEST_DIR)/TEST-Pyngraph.xml \ --ignore=$(INSTALL_TEST_DIR)/pyopenvino/tests/test_utils/test_utils.py \ --ignore=$(INSTALL_TEST_DIR)/pyopenvino/tests/test_onnx/test_zoo_models.py \ --ignore=$(INSTALL_TEST_DIR)/pyopenvino/tests/test_onnx/test_backend.py -v + env: + # Required by python imports to load requires libraries + # - tests install dir for mock_py + LD_LIBRARY_PATH: $(INSTALL_TEST_DIR) + # For python imports to import pybind_mock_frontend + PYTHONPATH: $(INSTALL_TEST_DIR) displayName: 'Python API 2.0 Tests' - script: | - export PYTHONPATH=$(INSTALL_DIR)/lib/python3/dist-packages:$PYTHONPATH - python3 -m pytest -s $(INSTALL_TEST_DIR)/mo/unit_tests --junitxml=$(INSTALL_TEST_DIR)/TEST-ModelOptimizer.xml + # TODO: fix 'No mock frontend API available' + $(TEST_VENV)/bin/python3 -m pip install openvino-dev --find-links=$(INSTALL_DIR)/tools + $(TEST_VENV)/bin/python3 -m pytest -s $(INSTALL_TEST_DIR)/mo/unit_tests --junitxml=$(INSTALL_TEST_DIR)/TEST-ModelOptimizer.xml displayName: 'Model Optimizer UT' # run not all smoke filter to save time in post-commit @@ -370,13 +372,10 @@ jobs: - script: cmake -DCOMPONENT=tests -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) -P $(BUILD_SAMPLES_TESTS_DIR)/cmake_install.cmake displayName: 'Install Samples Tests' - - script: python3 -m pip install -r $(INSTALL_TEST_DIR)/smoke_tests/requirements.txt - displayName: 'Install dependencies for samples smoke tests' - - script: | - export PATH=$HOME/.local/bin:$PATH + /usr/bin/python3 -m pip install -r $(INSTALL_TEST_DIR)/smoke_tests/requirements.txt # GNA isn't a part of Debian package, so filter out that tests - python3 -m pytest $(INSTALL_TEST_DIR)/smoke_tests/ -k "not GNA" --env_conf $(INSTALL_TEST_DIR)/smoke_tests/env_config.yml -s --junitxml=$(INSTALL_TEST_DIR)/TEST-SamplesSmokeTests.xml + /usr/bin/python3 -m pytest $(INSTALL_TEST_DIR)/smoke_tests/ -k "not GNA" --env_conf $(INSTALL_TEST_DIR)/smoke_tests/env_config.yml -s --junitxml=$(INSTALL_TEST_DIR)/TEST-SamplesSmokeTests.xml env: IE_APP_PATH: $(INSTALL_DIR)/samples_bin LD_LIBRARY_PATH: $(INSTALL_DIR)/samples_bin @@ -386,16 +385,18 @@ jobs: displayName: 'Samples Smoke Tests' - script: | - python3 -m pip install -r $(LAYER_TESTS_DIR)/requirements.txt - export PYTHONPATH=$(LAYER_TESTS_DIR):$PYTHONPATH - python3 -m pytest $(LAYER_TESTS_DIR)/tensorflow_tests/test_tf_Roll.py --ir_version=10 --junitxml=$(INSTALL_TEST_DIR)/TEST-tf_Roll.xmlTEST + $(TEST_VENV)/bin/python3 -m pip install -r $(LAYER_TESTS_DIR)/requirements.txt + $(TEST_VENV)/bin/python3 -m pytest $(LAYER_TESTS_DIR)/tensorflow_tests/test_tf_Roll.py --ir_version=10 --junitxml=$(INSTALL_TEST_DIR)/TEST-tf_Roll.xmlTEST + env: + PYTHONPATH: $(LAYER_TESTS_DIR) displayName: 'TensorFlow 1 Layer Tests - Legacy FE' - script: | - python3 -m pip install -r $(LAYER_TESTS_DIR)/requirements.txt - export PYTHONPATH=$(REPO_DIR)/tools/mo/:$(LAYER_TESTS_DIR):$PYTHONPATH - export TEST_DEVICE=CPU - $(RUN_PREFIX) python3 -m pytest $(LAYER_TESTS_DIR)/tensorflow_lite_tests/ --junitxml=$(INSTALL_TEST_DIR)/TEST-tfl_fe.xmlTEST + $(TEST_VENV)/bin/python3 -m pip install -r $(LAYER_TESTS_DIR)/requirements.txt + $(RUN_PREFIX) $(TEST_VENV)/bin/python3 -m pytest $(LAYER_TESTS_DIR)/tensorflow_lite_tests/ --junitxml=$(INSTALL_TEST_DIR)/TEST-tfl_fe.xmlTEST + env: + PYTHONPATH: $(REPO_DIR)/tools/mo/:$(LAYER_TESTS_DIR) + TEST_DEVICE: CPU displayName: 'TensorFlow Lite Layer Tests - TFL FE' - task: PublishTestResults@2 diff --git a/cmake/developer_package/packaging/debian/debian.cmake b/cmake/developer_package/packaging/debian/debian.cmake index f279ef5e4dc..51017cb7646 100644 --- a/cmake/developer_package/packaging/debian/debian.cmake +++ b/cmake/developer_package/packaging/debian/debian.cmake @@ -32,7 +32,7 @@ macro(ov_debian_cpack_set_dirs) set(OV_CPACK_OPENVINO_CMAKEDIR ${OV_CPACK_RUNTIMEDIR}/cmake/openvino${OpenVINO_VERSION}) set(OV_CPACK_DOCDIR ${CMAKE_INSTALL_DATADIR}/doc/openvino-${OpenVINO_VERSION}) set(OV_CPACK_LICENSESDIR ${OV_CPACK_DOCDIR}/licenses) - set(OV_CPACK_PYTHONDIR ${OV_CPACK_RUNTIMEDIR}/python3/dist-packages) + set(OV_CPACK_PYTHONDIR lib/python3/dist-packages) # non-native stuff set(OV_CPACK_SHAREDIR ${CMAKE_INSTALL_DATADIR}/openvino) # internal @@ -93,11 +93,21 @@ macro(ov_define_component_include_rules) # samples unset(OV_CPACK_COMP_CPP_SAMPLES_EXCLUDE_ALL) set(OV_CPACK_COMP_C_SAMPLES_EXCLUDE_ALL ${OV_CPACK_COMP_CPP_SAMPLES_EXCLUDE_ALL}) - set(OV_CPACK_COMP_PYTHON_SAMPLES_EXCLUDE_ALL EXCLUDE_FROM_ALL) + if(ENABLE_PYTHON_PACKAGING) + unset(OV_CPACK_COMP_PYTHON_SAMPLES_EXCLUDE_ALL) + else() + set(OV_CPACK_COMP_PYTHON_SAMPLES_EXCLUDE_ALL EXCLUDE_FROM_ALL) + endif() # python - set(OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL EXCLUDE_FROM_ALL) - set(OV_CPACK_COMP_PYTHON_IE_API_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) - set(OV_CPACK_COMP_PYTHON_NGRAPH_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) + if(ENABLE_PYTHON_PACKAGING) + unset(OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL) + set(OV_CPACK_COMP_PYTHON_IE_API_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) + set(OV_CPACK_COMP_PYTHON_NGRAPH_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) + else() + set(OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL EXCLUDE_FROM_ALL) + set(OV_CPACK_COMP_PYTHON_IE_API_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) + set(OV_CPACK_COMP_PYTHON_NGRAPH_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) + endif() set(OV_CPACK_COMP_PYTHON_WHEELS_EXCLUDE_ALL EXCLUDE_FROM_ALL) # tools set(OV_CPACK_COMP_CORE_TOOLS_EXCLUDE_ALL EXCLUDE_FROM_ALL) diff --git a/cmake/developer_package/packaging/rpm/rpm.cmake b/cmake/developer_package/packaging/rpm/rpm.cmake index 205600fd233..63caf2c115a 100644 --- a/cmake/developer_package/packaging/rpm/rpm.cmake +++ b/cmake/developer_package/packaging/rpm/rpm.cmake @@ -89,11 +89,21 @@ macro(ov_define_component_include_rules) # samples unset(OV_CPACK_COMP_CPP_SAMPLES_EXCLUDE_ALL) set(OV_CPACK_COMP_C_SAMPLES_EXCLUDE_ALL ${OV_CPACK_COMP_CPP_SAMPLES_EXCLUDE_ALL}) - set(OV_CPACK_COMP_PYTHON_SAMPLES_EXCLUDE_ALL EXCLUDE_FROM_ALL) + if(ENABLE_PYTHON_PACKAGING) + unset(OV_CPACK_COMP_PYTHON_SAMPLES_EXCLUDE_ALL) + else() + set(OV_CPACK_COMP_PYTHON_SAMPLES_EXCLUDE_ALL EXCLUDE_FROM_ALL) + endif() # python - set(OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL EXCLUDE_FROM_ALL) - set(OV_CPACK_COMP_PYTHON_IE_API_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) - set(OV_CPACK_COMP_PYTHON_NGRAPH_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) + if(ENABLE_PYTHON_PACKAGING) + unset(OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL) + set(OV_CPACK_COMP_PYTHON_IE_API_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) + set(OV_CPACK_COMP_PYTHON_NGRAPH_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) + else() + set(OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL EXCLUDE_FROM_ALL) + set(OV_CPACK_COMP_PYTHON_IE_API_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) + set(OV_CPACK_COMP_PYTHON_NGRAPH_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) + endif() set(OV_CPACK_COMP_PYTHON_WHEELS_EXCLUDE_ALL EXCLUDE_FROM_ALL) # tools set(OV_CPACK_COMP_CORE_TOOLS_EXCLUDE_ALL EXCLUDE_FROM_ALL) diff --git a/cmake/features.cmake b/cmake/features.cmake index b042e38cbf0..f372abe1a1b 100644 --- a/cmake/features.cmake +++ b/cmake/features.cmake @@ -175,6 +175,10 @@ ie_dependent_option (ENABLE_SYSTEM_PROTOBUF "Enables use of system Protobuf" OFF ie_dependent_option (ENABLE_SYSTEM_SNAPPY "Enables use of system version of Snappy" OFF "ENABLE_SNAPPY_COMPRESSION" OFF) +# temporary option until we enable this by default when review python API distribution +ie_dependent_option (ENABLE_PYTHON_PACKAGING "Enables packaging of Python API in APT / YUM" OFF + "ENABLE_PYTHON;CPACK_GENERATOR STREQUAL RPM OR CPACK_GENERATOR STREQUAL DEB" OFF) + ie_option(ENABLE_OPENVINO_DEBUG "Enable output for OPENVINO_DEBUG statements" OFF) if(NOT BUILD_SHARED_LIBS AND ENABLE_OV_TF_FRONTEND) diff --git a/cmake/packaging/debian.cmake b/cmake/packaging/debian.cmake index 407c5038492..7302127f3e3 100644 --- a/cmake/packaging/debian.cmake +++ b/cmake/packaging/debian.cmake @@ -47,8 +47,8 @@ macro(ov_cpack_settings) string(TOUPPER ${item} UPPER_COMP) # filter out some components, which are not needed to be wrapped to .deb package if(NOT OV_CPACK_COMP_${UPPER_COMP}_EXCLUDE_ALL AND - # skip OpenVINO Pyhon API and samples - NOT item MATCHES "^${OV_CPACK_COMP_PYTHON_OPENVINO}_python.*" AND + # skip OpenVINO Python API (pattern in form of "_python${PYTHON_VERSION_MAJOR}${PYTHON_VERSION_MINOR}") + (NOT item MATCHES "^${OV_CPACK_COMP_PYTHON_OPENVINO}_python.*" OR ENABLE_PYTHON_PACKAGING) AND # see ticket # 82605 NOT item STREQUAL "gna" AND # don't install Intel OpenMP during debian @@ -109,7 +109,7 @@ macro(ov_cpack_settings) # hetero if(ENABLE_HETERO) - set(CPACK_COMPONENT_HETERO_DESCRIPTION "OpenVINO Hetero plugin") + set(CPACK_COMPONENT_HETERO_DESCRIPTION "OpenVINO Hetero software plugin") set(CPACK_COMPONENT_HETERO_DEPENDS "${OV_CPACK_COMP_CORE}") set(CPACK_DEBIAN_HETERO_PACKAGE_NAME "libopenvino-hetero-plugin-${cpack_name_ver}") set(CPACK_DEBIAN_HETERO_PACKAGE_CONTROL_EXTRA "${def_postinst};${def_postrm}") @@ -119,7 +119,7 @@ macro(ov_cpack_settings) # auto batch if(ENABLE_AUTO_BATCH) - set(CPACK_COMPONENT_BATCH_DESCRIPTION "OpenVINO Automatic Batching plugin") + set(CPACK_COMPONENT_BATCH_DESCRIPTION "OpenVINO Automatic Batching software plugin") set(CPACK_COMPONENT_BATCH_DEPENDS "${OV_CPACK_COMP_CORE}") set(CPACK_DEBIAN_BATCH_PACKAGE_NAME "libopenvino-auto-batch-plugin-${cpack_name_ver}") set(CPACK_DEBIAN_BATCH_PACKAGE_CONTROL_EXTRA "${def_postinst};${def_postrm}") @@ -130,9 +130,9 @@ macro(ov_cpack_settings) # multi / auto plugins if(ENABLE_MULTI) if(ENABLE_AUTO) - set(CPACK_COMPONENT_MULTI_DESCRIPTION "OpenVINO Auto / Multi plugin") + set(CPACK_COMPONENT_MULTI_DESCRIPTION "OpenVINO Auto / Multi software plugin") else() - set(CPACK_COMPONENT_MULTI_DESCRIPTION "OpenVINO Multi plugin") + set(CPACK_COMPONENT_MULTI_DESCRIPTION "OpenVINO Multi software plugin") endif() set(CPACK_COMPONENT_MULTI_DEPENDS "${OV_CPACK_COMP_CORE}") set(CPACK_DEBIAN_MULTI_PACKAGE_NAME "libopenvino-auto-plugin-${cpack_name_ver}") @@ -140,7 +140,7 @@ macro(ov_cpack_settings) _ov_add_plugin(multi ON) set(multi_copyright "generic") elseif(ENABLE_AUTO) - set(CPACK_COMPONENT_AUTO_DESCRIPTION "OpenVINO Auto plugin") + set(CPACK_COMPONENT_AUTO_DESCRIPTION "OpenVINO Auto software plugin") set(CPACK_COMPONENT_AUTO_DEPENDS "${OV_CPACK_COMP_CORE}") set(CPACK_DEBIAN_AUTO_PACKAGE_NAME "libopenvino-auto-plugin-${cpack_name_ver}") set(CPACK_DEBIAN_AUTO_PACKAGE_CONTROL_EXTRA "${def_postinst};${def_postrm}") @@ -152,11 +152,11 @@ macro(ov_cpack_settings) if(ENABLE_INTEL_CPU) if(ARM OR AARCH64) set(CPACK_DEBIAN_CPU_PACKAGE_NAME "libopenvino-arm-cpu-plugin-${cpack_name_ver}") - set(CPACK_COMPONENT_CPU_DESCRIPTION "ARM® CPU plugin") + set(CPACK_COMPONENT_CPU_DESCRIPTION "ARM® CPU inference plugin") set(cpu_copyright "arm_cpu") elseif(X86 OR X86_64) set(CPACK_DEBIAN_CPU_PACKAGE_NAME "libopenvino-intel-cpu-plugin-${cpack_name_ver}") - set(CPACK_COMPONENT_CPU_DESCRIPTION "Intel® CPU plugin") + set(CPACK_COMPONENT_CPU_DESCRIPTION "Intel® CPU inference plugin") set(cpu_copyright "generic") else() message(FATAL_ERROR "Unsupported CPU architecture: ${CMAKE_SYSTEM_PROCESSOR}") @@ -168,19 +168,19 @@ macro(ov_cpack_settings) # intel-gpu if(ENABLE_INTEL_GPU) - set(CPACK_COMPONENT_GPU_DESCRIPTION "Intel® Processor Graphics plugin") + set(CPACK_COMPONENT_GPU_DESCRIPTION "Intel® Processor Graphics inference plugin") set(CPACK_COMPONENT_GPU_DEPENDS "${OV_CPACK_COMP_CORE}") set(CPACK_DEBIAN_GPU_PACKAGE_NAME "libopenvino-intel-gpu-plugin-${cpack_name_ver}") set(CPACK_DEBIAN_GPU_PACKAGE_CONTROL_EXTRA "${def_postinst};${def_postrm}") # auto batch exhances GPU - # set(CPACK_DEBIAN_BATCH_PACKAGE_ENHANCES "${CPACK_DEBIAN_GPU_PACKAGE_NAME} = (${cpack_full_ver})") + # set(CPACK_DEBIAN_BATCH_PACKAGE_ENHANCES "${CPACK_DEBIAN_GPU_PACKAGE_NAME} (= ${cpack_full_ver})") _ov_add_plugin(gpu OFF) set(gpu_copyright "generic") endif() # intel-gna if(ENABLE_INTEL_GNA AND "gna" IN_LIST CPACK_COMPONENTS_ALL) - set(CPACK_COMPONENT_GNA_DESCRIPTION "Intel® Gaussian Neural Accelerator") + set(CPACK_COMPONENT_GNA_DESCRIPTION "Intel® Gaussian Neural Accelerator inference plugin") set(CPACK_COMPONENT_GNA_DEPENDS "${OV_CPACK_COMP_CORE}") set(CPACK_DEBIAN_GNA_PACKAGE_NAME "libopenvino-intel-gna-plugin-${cpack_name_ver}") # since we have libgna.so we need to call ldconfig and have `def_triggers` here @@ -296,22 +296,25 @@ macro(ov_cpack_settings) set(${OV_CPACK_COMP_CORE_DEV}_copyright "generic") # - # Python bindings + # Python API # - if(ENABLE_PYTHON) + if(ENABLE_PYTHON_PACKAGING) ov_get_pyversion(pyversion) set(python_component "${OV_CPACK_COMP_PYTHON_OPENVINO}_${pyversion}") string(TOUPPER "${pyversion}" pyversion) - set(CPACK_COMPONENT_PYOPENVINO_${pyversion}_DESCRIPTION "OpenVINO Python bindings") + set(CPACK_COMPONENT_PYOPENVINO_${pyversion}_DESCRIPTION "OpenVINO Python API") set(CPACK_COMPONENT_PYOPENVINO_${pyversion}_DEPENDS "${OV_CPACK_COMP_CORE}") list(APPEND CPACK_COMPONENT_PYOPENVINO_${pyversion}_DEPENDS ${installed_plugins}) list(APPEND CPACK_COMPONENT_PYOPENVINO_${pyversion}_DEPENDS ${frontends}) - set(CPACK_DEBIAN_PYOPENVINO_${pyversion}_PACKAGE_NAME "libopenvino-python-${cpack_name_ver}") - set(CPACK_DEBIAN_PYOPENVINO_${pyversion}_PACKAGE_CONTROL_EXTRA "${def_postinst};${def_postrm}") - set(CPACK_DEBIAN_PYOPENVINO_${pyversion}_PACKAGE_DEPENDS "python3") + set(CPACK_DEBIAN_PYOPENVINO_${pyversion}_PACKAGE_NAME "python3-openvino") + set(python_package "${CPACK_DEBIAN_PYOPENVINO_${pyversion}_PACKAGE_NAME} (= ${cpack_full_ver})") + set(CPACK_DEBIAN_PYOPENVINO_${pyversion}_PACKAGE_DEPENDS "python3, python3-numpy") + + # we can have a single python installed, so we need to generate conflicts for all other versions + ov_debian_generate_conflicts(${python_component} ${conflicting_versions}) # TODO: fix all the warnings ov_debian_add_lintian_suppression(${python_component} @@ -352,11 +355,12 @@ macro(ov_cpack_settings) set(samples_copyright "generic") # python_samples - if(ENABLE_PYTHON) + if(ENABLE_PYTHON_PACKAGING) set(CPACK_COMPONENT_PYTHON_SAMPLES_DESCRIPTION "Intel(R) Distribution of OpenVINO(TM) Toolkit Python Samples") set(CPACK_COMPONENT_PYTHON_SAMPLES_DEPENDS "${python_component}") set(CPACK_DEBIAN_PYTHON_SAMPLES_PACKAGE_NAME "openvino-samples-python-${cpack_name_ver}") - set(CPACK_DEBIAN_PYTHON_SAMPLES_PACKAGE_DEPENDS "python3") + set(python_samples_package "${CPACK_DEBIAN_PYTHON_SAMPLES_PACKAGE_NAME} (= ${cpack_full_ver})") + set(CPACK_DEBIAN_PYTHON_SAMPLES_PACKAGE_DEPENDS "python3, ${python_package}") set(CPACK_DEBIAN_PYTHON_SAMPLES_PACKAGE_ARCHITECTURE "all") set(python_samples_copyright "generic") endif() @@ -387,6 +391,9 @@ macro(ov_cpack_settings) # all openvino set(CPACK_COMPONENT_OPENVINO_DESCRIPTION "Intel(R) Distribution of OpenVINO(TM) Toolkit Libraries and Development files") set(CPACK_COMPONENT_OPENVINO_DEPENDS "libraries_dev;${OV_CPACK_COMP_CPP_SAMPLES}") + if(ENABLE_PYTHON_PACKAGING) + list(APPEND CPACK_DEBIAN_OPENVINO_PACKAGE_DEPENDS "${python_package}, ${python_samples_package}") + endif() set(CPACK_DEBIAN_OPENVINO_PACKAGE_NAME "openvino-${cpack_name_ver}") set(CPACK_DEBIAN_OPENVINO_PACKAGE_ARCHITECTURE "all") ov_debian_generate_conflicts(openvino ${conflicting_versions}) diff --git a/cmake/packaging/rpm.cmake b/cmake/packaging/rpm.cmake index 44cba7292a3..4c9bbf71254 100644 --- a/cmake/packaging/rpm.cmake +++ b/cmake/packaging/rpm.cmake @@ -33,8 +33,8 @@ macro(ov_cpack_settings) string(TOUPPER ${item} UPPER_COMP) # filter out some components, which are not needed to be wrapped to .deb package if(NOT OV_CPACK_COMP_${UPPER_COMP}_EXCLUDE_ALL AND - # skip OpenVINO Pyhon API and samples - NOT item MATCHES "^${OV_CPACK_COMP_PYTHON_OPENVINO}_python.*" AND + # skip OpenVINO Python API (pattern in form of "_python${PYTHON_VERSION_MAJOR}${PYTHON_VERSION_MINOR}") + (NOT item MATCHES "^${OV_CPACK_COMP_PYTHON_OPENVINO}_python.*" OR ENABLE_PYTHON_PACKAGING) AND # see ticket # 82605 NOT item STREQUAL "gna" AND # don't install Intel OpenMP during rpm @@ -113,7 +113,7 @@ macro(ov_cpack_settings) # hetero if(ENABLE_HETERO) - set(CPACK_COMPONENT_HETERO_DESCRIPTION "OpenVINO Hetero plugin") + set(CPACK_COMPONENT_HETERO_DESCRIPTION "OpenVINO Hetero software plugin") set(CPACK_RPM_HETERO_PACKAGE_REQUIRES "${core_package}") set(CPACK_RPM_HETERO_PACKAGE_NAME "libopenvino-hetero-plugin-${cpack_name_ver}") _ov_add_package(plugin_packages hetero) @@ -122,7 +122,7 @@ macro(ov_cpack_settings) # auto batch if(ENABLE_AUTO_BATCH) - set(CPACK_COMPONENT_BATCH_DESCRIPTION "OpenVINO Automatic Batching plugin") + set(CPACK_COMPONENT_BATCH_DESCRIPTION "OpenVINO Automatic Batching software plugin") set(CPACK_RPM_BATCH_PACKAGE_REQUIRES "${core_package}") set(CPACK_RPM_BATCH_PACKAGE_NAME "libopenvino-auto-batch-plugin-${cpack_name_ver}") _ov_add_package(plugin_packages batch) @@ -132,16 +132,16 @@ macro(ov_cpack_settings) # multi / auto plugins if(ENABLE_MULTI) if(ENABLE_AUTO) - set(CPACK_COMPONENT_MULTI_DESCRIPTION "OpenVINO Auto / Multi plugin") + set(CPACK_COMPONENT_MULTI_DESCRIPTION "OpenVINO Auto / Multi software plugin") else() - set(CPACK_COMPONENT_MULTI_DESCRIPTION "OpenVINO Multi plugin") + set(CPACK_COMPONENT_MULTI_DESCRIPTION "OpenVINO Multi software plugin") endif() set(CPACK_RPM_MULTI_PACKAGE_REQUIRES "${core_package}") set(CPACK_RPM_MULTI_PACKAGE_NAME "libopenvino-auto-plugin-${cpack_name_ver}") _ov_add_package(plugin_packages multi) set(multi_copyright "generic") elseif(ENABLE_AUTO) - set(CPACK_COMPONENT_AUTO_DESCRIPTION "OpenVINO Auto plugin") + set(CPACK_COMPONENT_AUTO_DESCRIPTION "OpenVINO Auto software plugin") set(CPACK_RPM_AUTO_PACKAGE_REQUIRES "${core_package}") set(CPACK_RPM_AUTO_PACKAGE_NAME "libopenvino-auto-plugin-${cpack_name_ver}") _ov_add_package(plugin_packages auto) @@ -152,11 +152,11 @@ macro(ov_cpack_settings) if(ENABLE_INTEL_CPU) if(ARM OR AARCH64) set(CPACK_RPM_CPU_PACKAGE_NAME "libopenvino-arm-cpu-plugin-${cpack_name_ver}") - set(CPACK_COMPONENT_CPU_DESCRIPTION "ARM® CPU plugin") + set(CPACK_COMPONENT_CPU_DESCRIPTION "ARM® CPU inference plugin") set(cpu_copyright "arm_cpu") elseif(X86 OR X86_64) set(CPACK_RPM_CPU_PACKAGE_NAME "libopenvino-intel-cpu-plugin-${cpack_name_ver}") - set(CPACK_COMPONENT_CPU_DESCRIPTION "Intel® CPU") + set(CPACK_COMPONENT_CPU_DESCRIPTION "Intel® CPU inference plugin") set(cpu_copyright "generic") else() message(FATAL_ERROR "Unsupported CPU architecture: ${CMAKE_SYSTEM_PROCESSOR}") @@ -167,7 +167,7 @@ macro(ov_cpack_settings) # intel-gpu if(ENABLE_INTEL_GPU) - set(CPACK_COMPONENT_GPU_DESCRIPTION "Intel® Processor Graphics") + set(CPACK_COMPONENT_GPU_DESCRIPTION "Intel® Processor Graphics inference plugin") set(CPACK_RPM_GPU_PACKAGE_REQUIRES "${core_package}") set(CPACK_RPM_GPU_PACKAGE_NAME "libopenvino-intel-gpu-plugin-${cpack_name_ver}") _ov_add_package(plugin_packages gpu) @@ -176,7 +176,7 @@ macro(ov_cpack_settings) # intel-gna if(ENABLE_INTEL_GNA AND "gna" IN_LIST CPACK_COMPONENTS_ALL) - set(CPACK_COMPONENT_GNA_DESCRIPTION "Intel® Gaussian Neural Accelerator") + set(CPACK_COMPONENT_GNA_DESCRIPTION "Intel® Gaussian Neural Accelerator inference plugin") set(CPACK_RPM_GNA_PACKAGE_REQUIRES "${core_package}") set(CPACK_RPM_GNA_PACKAGE_NAME "libopenvino-intel-gna-plugin-${cpack_name_ver}") _ov_add_package(plugin_packages gna) @@ -264,17 +264,26 @@ macro(ov_cpack_settings) # Python bindings # - if(ENABLE_PYTHON) + if(ENABLE_PYTHON_PACKAGING) ov_get_pyversion(pyversion) set(python_component "${OV_CPACK_COMP_PYTHON_OPENVINO}_${pyversion}") - string(TOUPPER "${pyversion}" pyversion) + string(TOUPPER "${pyversion}" pyversion_upper) - set(CPACK_COMPONENT_PYOPENVINO_${pyversion}_DESCRIPTION "OpenVINO Python bindings") - set(CPACK_RPM_PYOPENVINO_${pyversion}_PACKAGE_REQUIRES - "${core_package}, ${frontend_packages}, ${plugin_packages}, python3") - set(CPACK_RPM_PYOPENVINO_${pyversion}_PACKAGE_NAME "libopenvino-python-${cpack_name_ver}") - set(python_package "${CPACK_RPM_PYOPENVINO_${pyversion}_PACKAGE_NAME} = ${cpack_full_ver}") + set(CPACK_COMPONENT_PYOPENVINO_${pyversion_upper}_DESCRIPTION "OpenVINO Python API") + set(CPACK_RPM_PYOPENVINO_${pyversion_upper}_PACKAGE_REQUIRES + "${core_package}, ${frontend_packages}, ${plugin_packages}, python3, python3-numpy") + set(CPACK_RPM_PYOPENVINO_${pyversion_upper}_PACKAGE_NAME "python3-openvino") + set(python_package "${CPACK_RPM_PYOPENVINO_${pyversion_upper}_PACKAGE_NAME} = ${cpack_full_ver}") set(${python_component}_copyright "generic") + + # we can have a single python installed, so we need to generate conflicts for all other versions + ov_rpm_generate_conflicts(${python_component} ${conflicting_versions}) + + ov_rpm_add_rpmlint_suppression("${python_component}" + # all directories + "non-standard-dir-perm /usr/lib64/${pyversion}/site-packages/openvino/*" + "non-standard-dir-perm /usr/lib64/${pyversion}/site-packages/ngraph/*" + ) endif() # @@ -309,12 +318,20 @@ macro(ov_cpack_settings) set(samples_copyright "generic") # python_samples - if(ENABLE_PYTHON) + if(ENABLE_PYTHON_PACKAGING) set(CPACK_COMPONENT_PYTHON_SAMPLES_DESCRIPTION "Intel(R) Distribution of OpenVINO(TM) Toolkit Python Samples") set(CPACK_RPM_PYTHON_SAMPLES_PACKAGE_REQUIRES "${python_package}, python3") set(CPACK_RPM_PYTHON_SAMPLES_PACKAGE_NAME "openvino-samples-python-${cpack_name_ver}") + set(python_samples_package "${CPACK_RPM_PYTHON_SAMPLES_PACKAGE_NAME} = ${cpack_full_ver}") set(CPACK_RPM_PYTHON_SAMPLES_PACKAGE_ARCHITECTURE "noarch") set(python_samples_copyright "generic") + + ov_rpm_add_rpmlint_suppression(${OV_CPACK_COMP_PYTHON_SAMPLES} + # all files + "non-executable-script /usr/share/openvino/samples/python/*" + # similar requirements.txt files + "files-duplicate /usr/share/openvino/samples/python/*" + ) endif() # @@ -345,6 +362,9 @@ macro(ov_cpack_settings) # all openvino set(CPACK_COMPONENT_OPENVINO_DESCRIPTION "Intel(R) Distribution of OpenVINO(TM) Toolkit Libraries and Development files") set(CPACK_RPM_OPENVINO_PACKAGE_REQUIRES "${libraries_dev_package}, ${samples_package}") + if(ENABLE_PYTHON_PACKAGING) + set(CPACK_DEBIAN_OPENVINO_PACKAGE_DEPENDS "${CPACK_RPM_OPENVINO_PACKAGE_REQUIRES}, ${python_package}, ${python_samples_package}") + endif() set(CPACK_RPM_OPENVINO_PACKAGE_NAME "openvino-${cpack_name_ver}") set(CPACK_RPM_OPENVINO_PACKAGE_ARCHITECTURE "noarch") ov_rpm_generate_conflicts(openvino ${conflicting_versions})