From 84a3aab11525a4d54487fcf347cac44838282bc7 Mon Sep 17 00:00:00 2001 From: Przemyslaw Wysocki Date: Mon, 7 Aug 2023 16:23:05 +0200 Subject: [PATCH] [PyOV] Backport of wheel building fix (#19013) * Add upper bound * backport flake fix * Support of protobuf >= 21 (#18351) * Corrected typo * Ability to compile with newer protobuf versions * Limit numpy (#18406) * Revert "[PyOV] Pin version of Cython for API 1.0 (#18604)" (#18681) * Revert "[PyOV] Pin version of Cython for API 1.0 (#18604)" This reverts commit 787796d88fb5c522d66f02273fb7841cedd1e1db. * Suppressed clang warning * Restrict scipy module version for POT (#18237) * Restrict scipy module version for POT Latest release https://pypi.org/project/scipy/1.11.0 causes dependency conflicts * Bump OMZ to include scipy restriction --------- Co-authored-by: Roman Kazantsev --------- Co-authored-by: Ilya Lavrenov Co-authored-by: Alina Kladieva Co-authored-by: Roman Kazantsev --- conanfile.txt | 4 ++-- samples/python/setup.cfg | 3 ++- src/bindings/python/constraints.txt | 2 +- .../openvino/inference_engine/CMakeLists.txt | 6 +++++- .../python/tests/test_onnx/test_ops_reshape.py | 2 +- src/bindings/python/wheel/requirements-dev.txt | 2 +- .../common/shutdown_protobuf/CMakeLists.txt | 1 + src/frontends/onnx/frontend/src/frontend.cpp | 12 +++++++++++- src/frontends/paddle/src/frontend.cpp | 12 +++++++++++- tests/constraints.txt | 2 +- thirdparty/CMakeLists.txt | 11 ++++++++++- thirdparty/onnx/CMakeLists.txt | 1 - thirdparty/open_model_zoo | 2 +- thirdparty/protobuf/CMakeLists.txt | 15 +++++++++++---- tools/mo/requirements_mxnet.txt | 2 +- tools/pot/setup.py | 2 +- 16 files changed, 60 insertions(+), 19 deletions(-) diff --git a/conanfile.txt b/conanfile.txt index 085464edcdb..2f6d597fbf0 100644 --- a/conanfile.txt +++ b/conanfile.txt @@ -2,7 +2,7 @@ ade/0.1.2a onetbb/[>=2021.2.1] pugixml/[>=1.10] -protobuf/[>=3.20.3] +protobuf/3.21.9 ittapi/[>=3.23.0] zlib/[>=1.2.8] opencl-icd-loader/[>=2022.09.30] @@ -19,7 +19,7 @@ flatbuffers/[>=22.9.24] [tool_requires] cmake/[>=3.15] patchelf/[>=0.12] -protobuf/[>=3.20.3] +protobuf/3.21.9 flatbuffers/[>=22.9.24] [options] diff --git a/samples/python/setup.cfg b/samples/python/setup.cfg index 1e8c86607e1..0326d23be63 100644 --- a/samples/python/setup.cfg +++ b/samples/python/setup.cfg @@ -3,9 +3,10 @@ # D100 - Missing docstring in public module # D101 - Missing docstring in public class # D103 - Missing docstring in public function +# VNE001 - Single letter variable names are not allowed filename = *.py max-line-length = 160 -ignore = E203,D100,D101,D103 +ignore = E203,D100,D101,D103,VNE001 max-parameters-amount = 8 show_source = True docstring-convention = google diff --git a/src/bindings/python/constraints.txt b/src/bindings/python/constraints.txt index 92b49d20cae..37730fb36ec 100644 --- a/src/bindings/python/constraints.txt +++ b/src/bindings/python/constraints.txt @@ -13,7 +13,7 @@ pytest-timeout==2.0.1 py>=1.9.0 pygments>=2.8.1 setuptools>=53.0.0 -wheel>=0.38.1 +wheel>=0.38.1,<=0.41.0 # Frontends docopt~=0.6.2 diff --git a/src/bindings/python/src/compatibility/openvino/inference_engine/CMakeLists.txt b/src/bindings/python/src/compatibility/openvino/inference_engine/CMakeLists.txt index 4a708026a24..51b884530e9 100644 --- a/src/bindings/python/src/compatibility/openvino/inference_engine/CMakeLists.txt +++ b/src/bindings/python/src/compatibility/openvino/inference_engine/CMakeLists.txt @@ -21,9 +21,13 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") # 'argument': conversion from 'size_t' to 'int', possible loss of data ie_add_compiler_flags(/wd4267) ie_add_compiler_flags(/wd4244) + ie_add_compiler_flags(/wd4551) endif() -if (OV_COMPILER_IS_CLANG OR CMAKE_COMPILER_IS_GNUCXX) +if(OV_COMPILER_IS_CLANG OR CMAKE_COMPILER_IS_GNUCXX) ie_add_compiler_flags(-Wno-undef) + if(OV_COMPILER_IS_CLANG) + ie_add_compiler_flags(-Wno-parentheses-equality) + endif() endif() if(UNUSED_BUT_SET_VARIABLE_SUPPORTED) ie_add_compiler_flags(-Wno-unused-but-set-variable) diff --git a/src/bindings/python/tests/test_onnx/test_ops_reshape.py b/src/bindings/python/tests/test_onnx/test_ops_reshape.py index 75670ac2039..c3da5e6c8a0 100644 --- a/src/bindings/python/tests/test_onnx/test_ops_reshape.py +++ b/src/bindings/python/tests/test_onnx/test_ops_reshape.py @@ -369,7 +369,7 @@ def test_split_1d(): def test_depth_to_space(): - b, c, h, w = shape = (2, 8, 3, 3) + b, c, h, w = shape = (2, 8, 3, 3) # noqa: VNE001 blocksize = 2 data = np.random.random_sample(shape).astype(np.float32) tmp = np.reshape(data, [b, blocksize, blocksize, c // (blocksize ** 2), h, w]) diff --git a/src/bindings/python/wheel/requirements-dev.txt b/src/bindings/python/wheel/requirements-dev.txt index 695e850d649..0e0f709690b 100644 --- a/src/bindings/python/wheel/requirements-dev.txt +++ b/src/bindings/python/wheel/requirements-dev.txt @@ -1,3 +1,3 @@ setuptools>=65.6.1 -wheel>=0.38.1 +wheel>=0.38.1,<=0.41.0 patchelf<=0.17.2.1; sys_platform == 'linux' and platform_machine == 'x86_64' diff --git a/src/frontends/common/shutdown_protobuf/CMakeLists.txt b/src/frontends/common/shutdown_protobuf/CMakeLists.txt index 4b0c2309abb..affac6ef605 100644 --- a/src/frontends/common/shutdown_protobuf/CMakeLists.txt +++ b/src/frontends/common/shutdown_protobuf/CMakeLists.txt @@ -9,3 +9,4 @@ add_library(${TARGET_NAME} STATIC shutdown_protobuf.cpp) target_include_directories(${TARGET_NAME} SYSTEM PRIVATE $>) set_target_properties(${TARGET_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO}) +target_compile_features(${TARGET_NAME} PRIVATE $) diff --git a/src/frontends/onnx/frontend/src/frontend.cpp b/src/frontends/onnx/frontend/src/frontend.cpp index 4108070c4f2..3ed28b64737 100644 --- a/src/frontends/onnx/frontend/src/frontend.cpp +++ b/src/frontends/onnx/frontend/src/frontend.cpp @@ -2,7 +2,15 @@ // SPDX-License-Identifier: Apache-2.0 // -#include +#include +#if PROTOBUF_VERSION >= 4022000 // protobuf 4.22 +# define OV_PROTOBUF_ABSL_IS_USED +#endif +#include + +#ifndef OV_PROTOBUF_ABSL_IS_USED +# include +#endif #include #include @@ -37,7 +45,9 @@ ONNX_FRONTEND_C_API void* get_front_end_data() { }; #ifndef OPENVINO_DEBUG_ENABLE // disable protobuf logging +# ifndef OV_PROTOBUF_ABSL_IS_USED google::protobuf::SetLogHandler(nullptr); +# endif #endif return res; } diff --git a/src/frontends/paddle/src/frontend.cpp b/src/frontends/paddle/src/frontend.cpp index 340f3e9722a..3ad7c9955de 100644 --- a/src/frontends/paddle/src/frontend.cpp +++ b/src/frontends/paddle/src/frontend.cpp @@ -4,7 +4,15 @@ #include "openvino/frontend/paddle/frontend.hpp" -#include +#include +#if PROTOBUF_VERSION >= 4022000 // protobuf 4.22 +# define OV_PROTOBUF_ABSL_IS_USED +#endif +#include + +#ifndef OV_PROTOBUF_ABSL_IS_USED +# include +#endif #include #include @@ -551,7 +559,9 @@ PADDLE_C_API void* get_front_end_data() { #ifndef OPENVINO_DEBUG_ENABLE // disable protobuf logging +# ifndef OV_PROTOBUF_ABSL_IS_USED google::protobuf::SetLogHandler(nullptr); +# endif #endif return res; } diff --git a/tests/constraints.txt b/tests/constraints.txt index 1b9bcfb8f87..200cb3e7baf 100644 --- a/tests/constraints.txt +++ b/tests/constraints.txt @@ -7,7 +7,7 @@ pandas>=1.3.5 pymongo>=3.12.0 PyYAML>=5.4.1 scipy>=1.7 -wheel>=0.38.1 +wheel>=0.38.1,<=0.41.0 defusedxml>=0.7.1 fastjsonschema~=2.15.1 test-generator==0.1.2 diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index 95d80a6fbd4..fdf25376ad8 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -423,7 +423,16 @@ if(ENABLE_OV_PADDLE_FRONTEND OR ENABLE_OV_ONNX_FRONTEND OR ENABLE_OV_TF_FRONTEND if(CMAKE_VERBOSE_MAKEFILE) set(Protobuf_DEBUG ON) endif() - find_package(Protobuf 3.20.3 REQUIRED) + # Note: we also specify 'protobuf' in NAMES because of vcpkg + set(find_package_protobuf_names Protobuf protobuf) + # try to find newer version first (major is changed) + # see https://protobuf.dev/support/version-support/ and + # https://github.com/protocolbuffers/protobuf/commit/d61f75ff6db36b4f9c0765f131f8edc2f86310fa + find_package(Protobuf 4.22.0 QUIET NAMES ${find_package_protobuf_names}) + if(NOT Protobuf_FOUND) + # otherwise, fallback to existing default + find_package(Protobuf 3.20.3 REQUIRED NAMES ${find_package_protobuf_names}) + endif() set(PROTOC_EXECUTABLE protobuf::protoc) # in case of system protobuf, we have to add version to OpenVINOConfig.cmake for static build diff --git a/thirdparty/onnx/CMakeLists.txt b/thirdparty/onnx/CMakeLists.txt index 1884c4b6464..3fb8cc3ea8f 100644 --- a/thirdparty/onnx/CMakeLists.txt +++ b/thirdparty/onnx/CMakeLists.txt @@ -49,7 +49,6 @@ endforeach() if(WIN32) # from onnx==1.13.1 it requires C++17 when compiling on Windows target_compile_features(onnx PRIVATE cxx_std_17) - set_target_properties(onnx PROPERTIES CXX_STANDARD 17) # OPTIONAL is a reserved word for mingw at least if(MINGW) target_compile_definitions(onnx PRIVATE OPTIONAL=OPTIONAL_PLACEHOLDER) diff --git a/thirdparty/open_model_zoo b/thirdparty/open_model_zoo index 5a513cc493f..c7a13a842d4 160000 --- a/thirdparty/open_model_zoo +++ b/thirdparty/open_model_zoo @@ -1 +1 @@ -Subproject commit 5a513cc493f4036fb7b543e952ec1a5fa72b74ee +Subproject commit c7a13a842d41333397ea6c3f9bc5a7053da00eec diff --git a/thirdparty/protobuf/CMakeLists.txt b/thirdparty/protobuf/CMakeLists.txt index 2f1fd6e4876..2ceda40a491 100644 --- a/thirdparty/protobuf/CMakeLists.txt +++ b/thirdparty/protobuf/CMakeLists.txt @@ -22,6 +22,7 @@ set(protobuf_VERBOSE ON) set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build tests" FORCE) set(protobuf_BUILD_SHARED_LIBS OFF CACHE BOOL "Build shared libs" FORCE) set(protobuf_WITH_ZLIB OFF CACHE BOOL "Build with zlib support" FORCE) +set(ABSL_PROPAGATE_CXX_STD ON CACHE BOOL "Abseil protogate CXX standard to dependent targets" FORCE) # note: HOST_AARCH64 AND X86_64 are not handled for Apple explicitly, because it can work via Rosetta if(CMAKE_CROSSCOMPILING OR @@ -38,8 +39,14 @@ if(NOT DEFINED protobuf_MSVC_STATIC_RUNTIME) set(protobuf_MSVC_STATIC_RUNTIME OFF CACHE BOOL "Link protobuf to static runtime libraries" FORCE) endif() -add_subdirectory(protobuf/cmake EXCLUDE_FROM_ALL) -get_directory_property(protobuf_VERSION DIRECTORY protobuf/cmake DEFINITION protobuf_VERSION) +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/protobuf/CMakeLists.txt") + set(protobuf_dir protobuf) +else() + set(protobuf_dir protobuf/cmake) +endif() + +add_subdirectory(${protobuf_dir} EXCLUDE_FROM_ALL) +get_directory_property(protobuf_VERSION DIRECTORY ${protobuf_dir} DEFINITION protobuf_VERSION) set(Protobuf_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/protobuf/src) @@ -61,7 +68,7 @@ if(CMAKE_COMPILER_IS_GNUCXX OR OV_COMPILER_IS_CLANG OR INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO}) endif() -if(protobuf_VERSION VERSION_LESS "3.9") +if(protobuf_VERSION VERSION_LESS 3.9) message(FATAL_ERROR "Minimum supported version of protobuf-lite library is 3.9.0 (provided ${protobuf_VERSION})") endif() @@ -76,7 +83,7 @@ if(NOT protobuf_BUILD_PROTOC_BINARIES) CMAKE_ARGS "-Dprotobuf_VERBOSE=${protobuf_VERBOSE}" "-Dprotobuf_BUILD_TESTS=${protobuf_BUILD_TESTS}" "-Dprotobuf_WITH_ZLIB=${protobuf_WITH_ZLIB}" - NATIVE_SOURCE_SUBDIR "protobuf/cmake" + NATIVE_SOURCE_SUBDIR "${protobuf_dir}" NATIVE_TARGETS protoc libprotobuf-lite) set(PROTOC_EXECUTABLE "${HOST_PROTOC_INSTALL_DIR}/bin/protoc") diff --git a/tools/mo/requirements_mxnet.txt b/tools/mo/requirements_mxnet.txt index 8df37622cc2..8b6561f761b 100644 --- a/tools/mo/requirements_mxnet.txt +++ b/tools/mo/requirements_mxnet.txt @@ -1,5 +1,5 @@ -c ../constraints.txt -numpy>=1.16.6,<1.25.0 +numpy>=1.16.6,<1.24 mxnet networkx defusedxml diff --git a/tools/pot/setup.py b/tools/pot/setup.py index 762c1923d12..45d69b50efd 100644 --- a/tools/pot/setup.py +++ b/tools/pot/setup.py @@ -67,7 +67,7 @@ if '--install-dev-extras' in sys.argv: INSTALL_REQUIRES = [ "numpy>=1.16.6", "scipy~=1.7; python_version == '3.7'", - "scipy>=1.8; python_version >= '3.8'", + "scipy>=1.8,<1.11; python_version >= '3.8'", "jstyleson>=0.0.2", "addict>=2.4.0", "networkx<=3.1",