diff --git a/.ci/azure/linux.yml b/.ci/azure/linux.yml index 3f619a386dc..b7143c1628f 100644 --- a/.ci/azure/linux.yml +++ b/.ci/azure/linux.yml @@ -354,7 +354,7 @@ jobs: - script: rm -fr $(BUILD_DIR) displayName: 'Clean build dir' - - script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ov_core_unit_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU* --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-NGraphUT.xml + - script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ov_core_unit_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-OVCoreUT.xml displayName: 'OV Core UT' - script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ov_inference_functional_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-InferenceFunc.xml @@ -363,6 +363,9 @@ jobs: - script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ov_inference_unit_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-InferenceUnit.xml displayName: 'Inference Unit Tests' + - script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-OVProxyTests.xml + displayName: 'OV Proxy Plugin Tests' + - script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ov_conditional_compilation_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-ConditionalCompilation.xml displayName: 'Conditional Compilation Tests' diff --git a/.ci/azure/linux_debian.yml b/.ci/azure/linux_debian.yml index 4b6a4475a3f..3ab575a84f0 100644 --- a/.ci/azure/linux_debian.yml +++ b/.ci/azure/linux_debian.yml @@ -276,6 +276,12 @@ jobs: LD_LIBRARY_PATH: $(INSTALL_TEST_DIR) displayName: 'OV Core UT' + - script: | + $(INSTALL_TEST_DIR)/ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-OVProxyTests.xml + env: + LD_LIBRARY_PATH: $(INSTALL_TEST_DIR) + displayName: 'OV Proxy Tests' + - script: $(INSTALL_TEST_DIR)/ov_onnx_frontend_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU* --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-ONNXFrontend.xml env: LD_LIBRARY_PATH: $(INSTALL_TEST_DIR) diff --git a/.ci/azure/mac.yml b/.ci/azure/mac.yml index 1ddebf0c522..236f3326c4a 100644 --- a/.ci/azure/mac.yml +++ b/.ci/azure/mac.yml @@ -180,10 +180,13 @@ jobs: - script: ls -alR $(INSTALL_DIR) displayName: 'List install files' - - script: . $(SETUPVARS) && $(INSTALL_TEST_DIR)/ov_core_unit_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU* --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-NGraphUT.xml + - script: . $(SETUPVARS) && $(INSTALL_TEST_DIR)/ov_core_unit_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU* --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-OVCoreUT.xml displayName: 'OV Core UT' enabled: 'false' + - script: . $(SETUPVARS) && $(INSTALL_TEST_DIR)/ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-OVProxyTests.xml + displayName: 'OV Proxy Plugin Tests' + - script: . $(SETUPVARS) && $(INSTALL_TEST_DIR)/ov_ir_frontend_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-IRFrontend.xml displayName: 'IR Frontend Tests' enabled: 'false' diff --git a/.ci/azure/windows.yml b/.ci/azure/windows.yml index fe67b1045d2..5396fcc915a 100644 --- a/.ci/azure/windows.yml +++ b/.ci/azure/windows.yml @@ -263,6 +263,9 @@ jobs: - script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\ov_inference_unit_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)\TEST-InferenceUnit.xml displayName: 'Inference Unit Tests' + - script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)\TEST-OVProxyTests.xml + displayName: 'OV Proxy Plugin Tests' + - script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\ov_conditional_compilation_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)\TEST-ConditionalCompilation.xml displayName: 'Conditional Compilation Tests' diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 683b8cfd620..22af2892bef 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -99,7 +99,10 @@ jobs: run: cmake -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/install_pkg -P '${{ github.workspace }}/build/cmake_install.cmake' - name: Run OV core unit tests - run: ${{ github.workspace }}/bin/intel64/Release/ov_core_unit_tests # --gtest_print_time=1 --gtest_filter=-*IE_GPU* --gtest_output=xml:${{ github.workspace }}/testdata/TEST-NGraphUT.xml + run: ${{ github.workspace }}/bin/intel64/Release/ov_core_unit_tests + + - name: Run OV Proxy plugin tests + run: ${{ github.workspace }}/bin/intel64/Release/ov_proxy_plugin_tests - name: Run IR frontend tests run: ${{ github.workspace }}/bin/intel64/Release/ov_ir_frontend_tests # --gtest_print_time=1 --gtest_output=xml:${{ github.workspace }}/testdata/TEST-IRFrontend.xml diff --git a/cmake/features.cmake b/cmake/features.cmake index 9a405208e0d..e0ab1b0b683 100644 --- a/cmake/features.cmake +++ b/cmake/features.cmake @@ -5,6 +5,7 @@ # # Common cmake options # +ov_option (ENABLE_PROXY "Proxy plugin for OpenVINO Runtime" ON) ie_dependent_option (ENABLE_INTEL_CPU "CPU plugin for OpenVINO Runtime" ON "RISCV64 OR X86 OR X86_64 OR AARCH64 OR ARM" OFF) diff --git a/docs/dev/cmake_options_for_custom_compilation.md b/docs/dev/cmake_options_for_custom_compilation.md index 30a5164ebbb..d79dcbb2c07 100644 --- a/docs/dev/cmake_options_for_custom_compilation.md +++ b/docs/dev/cmake_options_for_custom_compilation.md @@ -31,6 +31,8 @@ This document provides description and default values for CMake options that can * `ON` is default. * `ENABLE_AUTO_BATCH` enables Auto Batch plugin build: * `ON` is default. + * `ENABLE_PROXY` enables Proxy plugin compilation: + * `ON` is default. * Frontends to work with models from frameworks: * `ENABLE_OV_ONNX_FRONTEND` enables [ONNX] frontend plugin for OpenVINO Runtime: * `ON` is default. diff --git a/src/bindings/c/tests/ov_remote_context_test.cpp b/src/bindings/c/tests/ov_remote_context_test.cpp index b33faa1b501..abfd96d8d62 100644 --- a/src/bindings/c/tests/ov_remote_context_test.cpp +++ b/src/bindings/c/tests/ov_remote_context_test.cpp @@ -27,7 +27,7 @@ protected: char* info = nullptr; const char* key = ov_property_key_available_devices; EXPECT_EQ(ov_core_get_property(core, "GPU", key, &info), ov_status_e::OK); - EXPECT_STRNE(info, nullptr); + ASSERT_STRNE(info, nullptr); if (strlen(info) == 0) { ov_free(info); diff --git a/src/bindings/python/src/pyopenvino/core/core.cpp b/src/bindings/python/src/pyopenvino/core/core.cpp index 1dfe71c95f3..db04ee5bb76 100644 --- a/src/bindings/python/src/pyopenvino/core/core.cpp +++ b/src/bindings/python/src/pyopenvino/core/core.cpp @@ -485,11 +485,14 @@ void regclass_Core(py::module m) { new_compiled = core.import_model(user_stream, "CPU") )"); - cls.def("register_plugin", - &ov::Core::register_plugin, - py::arg("plugin_name"), - py::arg("device_name"), - R"( + cls.def( + "register_plugin", + [](ov::Core& self, const std::string& plugin_name, const std::string& device_name) { + self.register_plugin(plugin_name, device_name); + }, + py::arg("plugin_name"), + py::arg("device_name"), + R"( Register a new device and plugin which enable this device inside OpenVINO Runtime. :param plugin_name: A path (absolute or relative) or name of a plugin. Depending on platform, @@ -501,6 +504,32 @@ void regclass_Core(py::module m) { :type device_name: str )"); + cls.def( + "register_plugin", + [](ov::Core& self, + const std::string& plugin_name, + const std::string& device_name, + const std::map& config) { + auto properties = Common::utils::properties_to_any_map(config); + self.register_plugin(plugin_name, device_name, properties); + }, + py::arg("plugin_name"), + py::arg("device_name"), + py::arg("config"), + R"( + Register a new device and plugin which enable this device inside OpenVINO Runtime. + + :param plugin_name: A path (absolute or relative) or name of a plugin. Depending on platform, + `plugin_name` is wrapped with shared library suffix and prefix to identify + library full name E.g. on Linux platform plugin name specified as `plugin_name` + will be wrapped as `libplugin_name.so`. + :type plugin_name: str + :param device_name: A device name to register plugin for. + :type device_name: str + :param config: Plugin default configuration + :type config: dict, optional + )"); + cls.def("register_plugins", &ov::Core::register_plugins, py::arg("xml_config_file"), diff --git a/src/cmake/openvino.cmake b/src/cmake/openvino.cmake index 78b10ca0e2e..5e32896e938 100644 --- a/src/cmake/openvino.cmake +++ b/src/cmake/openvino.cmake @@ -25,7 +25,8 @@ add_library(${TARGET_NAME} $ $ $ - $) + $ + $<$:$>) add_library(openvino::runtime ALIAS ${TARGET_NAME}) set_target_properties(${TARGET_NAME} PROPERTIES EXPORT_NAME runtime) diff --git a/src/core/include/openvino/runtime/tensor.hpp b/src/core/include/openvino/runtime/tensor.hpp index 29ce1a0ceb3..4dee04d70f0 100644 --- a/src/core/include/openvino/runtime/tensor.hpp +++ b/src/core/include/openvino/runtime/tensor.hpp @@ -35,6 +35,10 @@ class IVariableStateInternalWrapper; class ITensor; class RemoteTensor; +namespace proxy { +class RemoteContext; +} + namespace op { namespace util { class VariableValue; @@ -68,6 +72,7 @@ protected: friend class ov::ISyncInferRequest; friend class ov::IInferRequestInternalWrapper; friend class ov::IVariableStateInternalWrapper; + friend class ov::proxy::RemoteContext; friend class InferenceEngine::IAsyncInferRequestWrapper; friend class InferenceEngine::IVariableStateWrapper; friend class ov::op::util::VariableValue; diff --git a/src/inference/CMakeLists.txt b/src/inference/CMakeLists.txt index 835b0d17576..d50bc84013a 100644 --- a/src/inference/CMakeLists.txt +++ b/src/inference/CMakeLists.txt @@ -132,6 +132,7 @@ ie_faster_build(${TARGET_NAME}_obj target_compile_definitions(${TARGET_NAME}_obj PRIVATE IMPLEMENT_INFERENCE_ENGINE_API + $<$:PROXY_PLUGIN_ENABLED> $ $ $) @@ -140,6 +141,7 @@ target_include_directories(${TARGET_NAME}_obj SYSTEM PRIVATE $ $ $ + $<$:$> $<$:$>) target_include_directories(${TARGET_NAME}_obj PRIVATE @@ -179,6 +181,7 @@ ov_ncc_naming_style(FOR_TARGET ${TARGET_NAME}_obj # Static library used for unit tests which are always built add_library(${TARGET_NAME}_s STATIC EXCLUDE_FROM_ALL + $<$:$> $) set_ie_threading_interface_for(${TARGET_NAME}_s) diff --git a/src/inference/dev_api/openvino/runtime/icore.hpp b/src/inference/dev_api/openvino/runtime/icore.hpp index 00f94c2dfb7..d0c4fd6d954 100644 --- a/src/inference/dev_api/openvino/runtime/icore.hpp +++ b/src/inference/dev_api/openvino/runtime/icore.hpp @@ -18,6 +18,12 @@ namespace ov { +namespace proxy { + +class Plugin; + +} + /** * @interface ICore * @brief Minimal ICore interface to allow plugin to get information from Core OpenVINO class. @@ -227,6 +233,10 @@ public: * @brief Default virtual destructor */ virtual ~ICore(); + +private: + virtual void set_property(const std::string& device_name, const AnyMap& properties) = 0; + friend class ov::proxy::Plugin; }; } // namespace ov diff --git a/src/inference/include/ie/gpu/gpu_context_api_dx.hpp b/src/inference/include/ie/gpu/gpu_context_api_dx.hpp index 368b7fd6149..bdc7b62a752 100644 --- a/src/inference/include/ie/gpu/gpu_context_api_dx.hpp +++ b/src/inference/include/ie/gpu/gpu_context_api_dx.hpp @@ -150,7 +150,7 @@ INFERENCE_ENGINE_1_0_DEPRECATED static inline D3DContext::Ptr make_shared_contex {GPU_PARAM_KEY(TILE_ID), target_tile_id} }; // clang-format on - return std::dynamic_pointer_cast(core.CreateContext(deviceName, contextParams)); + return std::dynamic_pointer_cast(core.CreateContext(deviceName, contextParams)->GetHardwareContext()); } /** @@ -163,7 +163,7 @@ INFERENCE_ENGINE_1_0_DEPRECATED static inline D3DContext::Ptr make_shared_contex INFERENCE_ENGINE_1_0_DEPRECATED static inline Blob::Ptr make_shared_blob(const TensorDesc& desc, RemoteContext::Ptr ctx, ID3D11Buffer* buffer) { - auto casted = std::dynamic_pointer_cast(ctx); + auto casted = ctx->as(); if (nullptr == casted) { IE_THROW() << "Invalid remote context passed"; } @@ -186,7 +186,7 @@ INFERENCE_ENGINE_1_0_DEPRECATED static inline Blob::Ptr make_shared_blob(const T RemoteContext::Ptr ctx, ID3D11Texture2D* surface, uint32_t plane = 0) { - auto casted = std::dynamic_pointer_cast(ctx); + auto casted = ctx->as(); if (nullptr == casted) { IE_THROW() << "Invalid remote context passed"; } diff --git a/src/inference/include/ie/gpu/gpu_context_api_ocl.hpp b/src/inference/include/ie/gpu/gpu_context_api_ocl.hpp index 2ad89c38a31..6de24c6499a 100644 --- a/src/inference/include/ie/gpu/gpu_context_api_ocl.hpp +++ b/src/inference/include/ie/gpu/gpu_context_api_ocl.hpp @@ -298,7 +298,7 @@ INFERENCE_ENGINE_1_0_DEPRECATED static inline Blob::Ptr make_shared_blob(const T INFERENCE_ENGINE_1_0_DEPRECATED static inline Blob::Ptr make_shared_blob(const TensorDesc& desc, RemoteContext::Ptr ctx, cl::Buffer& buffer) { - auto casted = std::dynamic_pointer_cast(ctx); + auto casted = ctx->as(); if (nullptr == casted) { IE_THROW() << "Invalid remote context passed"; } @@ -318,7 +318,7 @@ INFERENCE_ENGINE_1_0_DEPRECATED static inline Blob::Ptr make_shared_blob(const T INFERENCE_ENGINE_1_0_DEPRECATED static inline Blob::Ptr make_shared_blob(const TensorDesc& desc, RemoteContext::Ptr ctx, cl_mem buffer) { - auto casted = std::dynamic_pointer_cast(ctx); + auto casted = ctx->as(); if (nullptr == casted) { IE_THROW() << "Invalid remote context passed"; } @@ -338,7 +338,7 @@ INFERENCE_ENGINE_1_0_DEPRECATED static inline Blob::Ptr make_shared_blob(const T INFERENCE_ENGINE_1_0_DEPRECATED static inline Blob::Ptr make_shared_blob(const TensorDesc& desc, RemoteContext::Ptr ctx, cl::Image2D& image) { - auto casted = std::dynamic_pointer_cast(ctx); + auto casted = ctx->as(); if (nullptr == casted) { IE_THROW() << "Invalid remote context passed"; } diff --git a/src/inference/include/ie/gpu/gpu_context_api_va.hpp b/src/inference/include/ie/gpu/gpu_context_api_va.hpp index 6c6e2bf8adb..03a43d6f96d 100644 --- a/src/inference/include/ie/gpu/gpu_context_api_va.hpp +++ b/src/inference/include/ie/gpu/gpu_context_api_va.hpp @@ -117,7 +117,7 @@ INFERENCE_ENGINE_1_0_DEPRECATED static inline VAContext::Ptr make_shared_context ParamMap contextParams = {{GPU_PARAM_KEY(CONTEXT_TYPE), GPU_PARAM_VALUE(VA_SHARED)}, {GPU_PARAM_KEY(VA_DEVICE), static_cast(device)}, {GPU_PARAM_KEY(TILE_ID), target_tile_id}}; - return std::dynamic_pointer_cast(core.CreateContext(deviceName, contextParams)); + return std::dynamic_pointer_cast(core.CreateContext(deviceName, contextParams)->GetHardwareContext()); } /** @@ -132,7 +132,7 @@ INFERENCE_ENGINE_1_0_DEPRECATED static inline VASurfaceBlob::Ptr make_shared_blo RemoteContext::Ptr ctx, VASurfaceID surface, uint32_t plane = 0) { - auto casted = std::dynamic_pointer_cast(ctx); + auto casted = ctx->as(); if (nullptr == casted) { IE_THROW() << "Invalid remote context passed"; } diff --git a/src/inference/include/ie/ie_blob.h b/src/inference/include/ie/ie_blob.h index 18a83ecc68f..0a40427f316 100644 --- a/src/inference/include/ie/ie_blob.h +++ b/src/inference/include/ie/ie_blob.h @@ -40,6 +40,8 @@ namespace InferenceEngine { IE_SUPPRESS_DEPRECATED_START +class RemoteBlob; + /** * @brief This class represents a universal container in the Inference Engine * @@ -80,7 +82,7 @@ public: typename std::enable_if::value && !std::is_reference::value, int>::type = 0, typename std::enable_if::value, int>::type = 0> bool is() noexcept { - return dynamic_cast(this) != nullptr; + return dynamic_cast(getHardwareBlob()) != nullptr; } /** @@ -93,7 +95,7 @@ public: typename std::enable_if::value && !std::is_reference::value, int>::type = 0, typename std::enable_if::value, int>::type = 0> bool is() const noexcept { - return dynamic_cast(this) != nullptr; + return dynamic_cast(getHardwareBlob()) != nullptr; } /** @@ -104,9 +106,25 @@ public: * @tparam T Type to cast to. Must represent a class derived from the Blob * @return Raw pointer to the object of the type T or nullptr on error */ + template < + typename T, + typename std::enable_if::value && !std::is_reference::value, int>::type = 0, + typename std::enable_if::value && !std::is_same::value, int>::type = 0> + T* as() noexcept { + return dynamic_cast(getHardwareBlob()); + } + + /** + * @brief Casts this Blob object to the type RemoteBlob. + * + * Use InferenceEngine::as() to operate with shared Blob objects instead of raw pointers + * + * @tparam T Type to cast to. Must represent a class derived from the Blob + * @return Raw pointer to the object of the type T or nullptr on error + */ template ::value && !std::is_reference::value, int>::type = 0, - typename std::enable_if::value, int>::type = 0> + typename std::enable_if::value, int>::type = 0> T* as() noexcept { return dynamic_cast(this); } @@ -119,11 +137,27 @@ public: * @tparam T Type to cast to. Must represent a class derived from the Blob * @return Raw pointer to the object of the type const T or nullptr on error */ + template < + typename T, + typename std::enable_if::value && !std::is_reference::value, int>::type = 0, + typename std::enable_if::value && !std::is_same::value, int>::type = 0> + const T* as() const noexcept { + return dynamic_cast(getHardwareBlob()); + } + + /** + * @brief Casts this Blob object to the type RemoteBlob. + * + * Use InferenceEngine::as() to operate with shared Blob objects instead of raw pointers + * + * @tparam T Type to cast to. Must represent a class derived from the Blob + * @return Raw pointer to the object of the type T or nullptr on error + */ template ::value && !std::is_reference::value, int>::type = 0, - typename std::enable_if::value, int>::type = 0> + typename std::enable_if::value, int>::type = 0> const T* as() const noexcept { - return dynamic_cast(this); + return dynamic_cast(this); } /** @@ -286,6 +320,9 @@ protected: * @return The allocator for allocator-based blobs or nullptr if there is none */ virtual const std::shared_ptr& getAllocator() const noexcept = 0; + + const Blob* getHardwareBlob() const; + Blob* getHardwareBlob(); }; /** diff --git a/src/inference/include/ie/ie_remote_context.hpp b/src/inference/include/ie/ie_remote_context.hpp index 5485b278d4a..7e74e8c48ec 100644 --- a/src/inference/include/ie/ie_remote_context.hpp +++ b/src/inference/include/ie/ie_remote_context.hpp @@ -63,7 +63,7 @@ public: typename std::enable_if::value && !std::is_reference::value, int>::type = 0, typename std::enable_if::value, int>::type = 0> bool is() noexcept { - return dynamic_cast(this) != nullptr; + return dynamic_cast(GetHardwareContext().get()) != nullptr; } /** @@ -76,7 +76,7 @@ public: typename std::enable_if::value && !std::is_reference::value, int>::type = 0, typename std::enable_if::value, int>::type = 0> bool is() const noexcept { - return dynamic_cast(this) != nullptr; + return dynamic_cast(GetHardwareContext().get()) != nullptr; } /** @@ -89,7 +89,7 @@ public: typename std::enable_if::value && !std::is_reference::value, int>::type = 0, typename std::enable_if::value, int>::type = 0> T* as() noexcept { - return dynamic_cast(this); + return dynamic_cast(GetHardwareContext().get()); } /** @@ -102,7 +102,7 @@ public: typename std::enable_if::value && !std::is_reference::value, int>::type = 0, typename std::enable_if::value, int>::type = 0> const T* as() const noexcept { - return dynamic_cast(this); + return dynamic_cast(GetHardwareContext().get()); } /** @@ -141,6 +141,20 @@ public: * @return A map of name/parameter elements. */ virtual ParamMap getParams() const = 0; + + /** + * @brief Unwrap hardware remote context + * + * @return shared pointer to plugin specific remote context + */ + const std::shared_ptr GetHardwareContext(); + + /** + * @brief Unwrap hardware remote context + * + * @return shared pointer to plugin specific remote context + */ + const std::shared_ptr GetHardwareContext() const; }; /** diff --git a/src/inference/include/openvino/runtime/core.hpp b/src/inference/include/openvino/runtime/core.hpp index 1385ccdeffd..153559c17e0 100644 --- a/src/inference/include/openvino/runtime/core.hpp +++ b/src/inference/include/openvino/runtime/core.hpp @@ -688,8 +688,9 @@ public: * @note For security purposes it suggested to specify absolute path to register plugin. * * @param device_name Device name to register a plugin for. + * @param config Plugin configuration options */ - void register_plugin(const std::string& plugin, const std::string& device_name); + void register_plugin(const std::string& plugin, const std::string& device_name, const ov::AnyMap& config = {}); /** * @brief Unloads the previously loaded plugin identified by @p device_name from OpenVINO Runtime. diff --git a/src/inference/include/openvino/runtime/remote_context.hpp b/src/inference/include/openvino/runtime/remote_context.hpp index 6de4686cc57..f722a708610 100644 --- a/src/inference/include/openvino/runtime/remote_context.hpp +++ b/src/inference/include/openvino/runtime/remote_context.hpp @@ -37,6 +37,10 @@ class IExecutableNetworkWrapper; class ICompiledModel; class CompiledModel; +namespace proxy { +class RemoteContext; +} + /** * @brief This class represents an abstraction * @ingroup ov_runtime_cpp_api @@ -68,6 +72,7 @@ protected: friend class ov::IExecutableNetworkWrapper; friend class ov::ICompiledModel; friend class ov::CompiledModel; + friend class ov::proxy::RemoteContext; public: /** diff --git a/src/inference/include/openvino/runtime/remote_tensor.hpp b/src/inference/include/openvino/runtime/remote_tensor.hpp index 938398a07be..78a07b7c066 100644 --- a/src/inference/include/openvino/runtime/remote_tensor.hpp +++ b/src/inference/include/openvino/runtime/remote_tensor.hpp @@ -14,6 +14,10 @@ namespace ov { +namespace proxy { +class RemoteTensor; +} + class RemoteContext; /** @@ -23,6 +27,7 @@ class RemoteContext; class OPENVINO_RUNTIME_API RemoteTensor : public Tensor { using Tensor::Tensor; friend class ov::RemoteContext; + friend class ov::proxy::RemoteTensor; public: /** diff --git a/src/inference/src/core.cpp b/src/inference/src/core.cpp index 6d061db7280..2b2ed684682 100644 --- a/src/inference/src/core.cpp +++ b/src/inference/src/core.cpp @@ -253,8 +253,8 @@ std::vector Core::get_available_devices() const { OV_CORE_CALL_STATEMENT(return _impl->GetAvailableDevices();); } -void Core::register_plugin(const std::string& plugin, const std::string& device_name) { - OV_CORE_CALL_STATEMENT(_impl->register_plugin(plugin, device_name);); +void Core::register_plugin(const std::string& plugin, const std::string& device_name, const ov::AnyMap& properties) { + OV_CORE_CALL_STATEMENT(_impl->register_plugin(plugin, device_name, properties);); } void Core::unload_plugin(const std::string& device_name) { diff --git a/src/inference/src/dev/converter_utils.cpp b/src/inference/src/dev/converter_utils.cpp index 522d9abf2e8..dcbe4b12b41 100644 --- a/src/inference/src/dev/converter_utils.cpp +++ b/src/inference/src/dev/converter_utils.cpp @@ -39,9 +39,14 @@ #include "openvino/runtime/tensor.hpp" #include "openvino/runtime/threading/executor_manager.hpp" #include "openvino/runtime/variable_state.hpp" +#include "remote_context_wrapper.hpp" #include "threading/ie_executor_manager.hpp" #include "transformations/utils/utils.hpp" +#ifdef PROXY_PLUGIN_ENABLED +# include "openvino/proxy/infer_request.hpp" +#endif + namespace { std::string get_legacy_name_from_port(const ov::Output& port) { @@ -542,6 +547,12 @@ public: } const InferenceEngine::PreProcessInfo& GetPreProcess(const std::string& name) const override { +#ifdef PROXY_PLUGIN_ENABLED + if (auto proxy_request = std::dynamic_pointer_cast(m_request)) { + return ov::legacy_convert::convert_infer_request(proxy_request->get_hardware_request()._ptr) + ->GetPreProcess(name); + } +#endif OPENVINO_NOT_IMPLEMENTED; } @@ -613,9 +624,12 @@ public: class IAsyncInferRequestWrapper : public ov::IAsyncInferRequest { public: - IAsyncInferRequestWrapper(const std::shared_ptr& request) + IAsyncInferRequestWrapper(const std::shared_ptr& request, + const std::string& plugin_name) : ov::IAsyncInferRequest(nullptr, nullptr, nullptr), - m_request(request) { + m_request(request), + m_unwrap_tensor(plugin_name != "AUTO" && plugin_name != "MULTI" && plugin_name != "BATCH" && + plugin_name != "HETERO") { if (m_request->getPointerToExecutableNetworkInternal()) m_compiled_model = ov::legacy_convert::convert_compiled_model(m_request->getPointerToExecutableNetworkInternal()); @@ -708,7 +722,7 @@ public: return tensor; } void set_tensor(const ov::Output& port, const ov::Tensor& tensor) override { - m_request->SetBlob(get_legacy_name_from_port(port), ov::tensor_to_blob(tensor._impl)); + m_request->SetBlob(get_legacy_name_from_port(port), ov::tensor_to_blob(tensor._impl, m_unwrap_tensor)); } std::vector get_tensors(const ov::Output& port) const override { @@ -724,7 +738,7 @@ public: void set_tensors(const ov::Output& port, const std::vector& tensors) override { std::vector blobs; for (const auto& tensor : tensors) { - blobs.emplace_back(ov::tensor_to_blob(tensor._impl)); + blobs.emplace_back(ov::tensor_to_blob(tensor._impl, m_unwrap_tensor)); } m_request->SetBlobs(get_legacy_name_from_port(port), blobs); } @@ -766,6 +780,7 @@ private: std::shared_ptr m_request; mutable std::shared_ptr m_compiled_model; mutable std::mutex m_mutex; + const bool m_unwrap_tensor; }; } // namespace InferenceEngine @@ -778,51 +793,14 @@ std::shared_ptr<::InferenceEngine::IInferRequestInternal> ov::legacy_convert::co return std::make_shared(request); } std::shared_ptr<::ov::IAsyncInferRequest> ov::legacy_convert::convert_infer_request( - const std::shared_ptr<::InferenceEngine::IInferRequestInternal>& request) { + const std::shared_ptr<::InferenceEngine::IInferRequestInternal>& request, + const std::string& plugin_name) { if (auto comp_model = std::dynamic_pointer_cast(request)) { return comp_model->get_infer_request(); } - return std::make_shared(request); + return std::make_shared(request, plugin_name); } -namespace ov { - -class RemoteContextWrapper : public InferenceEngine::RemoteContext { -private: - std::shared_ptr m_context; - -public: - RemoteContextWrapper(const std::shared_ptr& context) : m_context(context) {} - - const std::shared_ptr& get_context() { - return m_context; - } - - std::string getDeviceName() const noexcept override { - return m_context->get_device_name(); - } - - InferenceEngine::RemoteBlob::Ptr CreateBlob(const InferenceEngine::TensorDesc& tensorDesc, - const InferenceEngine::ParamMap& params = {}) override { - return std::dynamic_pointer_cast(ov::tensor_to_blob( - m_context->create_tensor(InferenceEngine::details::convertPrecision(tensorDesc.getPrecision()), - tensorDesc.getBlockingDesc().getBlockDims(), - params))); - } - - InferenceEngine::MemoryBlob::Ptr CreateHostBlob(const InferenceEngine::TensorDesc& tensorDesc) override { - return std::dynamic_pointer_cast(ov::tensor_to_blob( - m_context->create_host_tensor(InferenceEngine::details::convertPrecision(tensorDesc.getPrecision()), - tensorDesc.getBlockingDesc().getBlockDims()))); - } - - InferenceEngine::ParamMap getParams() const override { - return m_context->get_property(); - } -}; - -} // namespace ov - namespace InferenceEngine { class IRemoteContextWrapper : public ov::IRemoteContext { diff --git a/src/inference/src/dev/converter_utils.hpp b/src/inference/src/dev/converter_utils.hpp index 2565ab8e3db..bd0dc77604b 100644 --- a/src/inference/src/dev/converter_utils.hpp +++ b/src/inference/src/dev/converter_utils.hpp @@ -34,7 +34,8 @@ std::shared_ptr<::ov::ICompiledModel> convert_compiled_model( std::shared_ptr<::InferenceEngine::IInferRequestInternal> convert_infer_request( const std::shared_ptr<::ov::IAsyncInferRequest>& request); std::shared_ptr<::ov::IAsyncInferRequest> convert_infer_request( - const std::shared_ptr<::InferenceEngine::IInferRequestInternal>& request); + const std::shared_ptr<::InferenceEngine::IInferRequestInternal>& request, + const std::string& plugin_name = ""); std::shared_ptr convert_remote_context( const std::shared_ptr& context); diff --git a/src/inference/src/dev/core_impl.cpp b/src/inference/src/dev/core_impl.cpp index 26eea32f6dd..e92d744f5f9 100644 --- a/src/inference/src/dev/core_impl.cpp +++ b/src/inference/src/dev/core_impl.cpp @@ -39,12 +39,18 @@ #include "openvino/util/shared_object.hpp" #include "ov_plugins.hpp" #include "preprocessing/preprocessing.hpp" +#ifdef PROXY_PLUGIN_ENABLED +# include "openvino/proxy/plugin.hpp" +# include "openvino/proxy/properties.hpp" +#endif #include "xml_parse_utils.h" ov::ICore::~ICore() = default; namespace { +static constexpr const char* internal_plugin_suffix = "_ov_internal"; + template void allowNotImplemented(F&& f) { try { @@ -309,6 +315,136 @@ ov::CoreImpl::CoreImpl(bool _newAPI) : m_new_api(_newAPI) { } } +bool ov::CoreImpl::is_proxy_device(const ov::Plugin& plugin) const { + return is_proxy_device(plugin.get_name()); +} +bool ov::CoreImpl::is_proxy_device(const std::string& dev_name) const { +#ifdef PROXY_PLUGIN_ENABLED + return pluginRegistry.find(dev_name) != pluginRegistry.end() && + pluginRegistry.at(dev_name).pluginCreateFunc == ov::proxy::create_plugin; +#else + return false; +#endif +} + +void ov::CoreImpl::register_plugin_in_registry_unsafe(const std::string& device_name, PluginDescriptor& desc) { +#ifdef PROXY_PLUGIN_ENABLED + // Update proxy plugin config + const auto& fill_config = [](ov::AnyMap& defaultConfig, const ov::AnyMap& config, const std::string& dev_name) { + // Configure aliases for proxy plugin + auto it = config.find(ov::proxy::configuration::alias.name()); + std::string alias; + if (it != config.end()) { + alias = it->second.as(); + if (defaultConfig.find(ov::proxy::alias_for.name()) == defaultConfig.end()) { + defaultConfig[ov::proxy::alias_for.name()] = std::vector(); + } + defaultConfig[ov::proxy::alias_for.name()].as>().emplace_back(dev_name); + } + + // Configure device order for proxy_plugin + it = config.find(ov::proxy::configuration::priority.name()); + if (it != config.end()) { + if (defaultConfig.find(ov::proxy::device_priorities.name()) == defaultConfig.end()) { + defaultConfig[ov::proxy::device_priorities.name()] = std::vector(); + } + defaultConfig[ov::proxy::device_priorities.name()].as>().emplace_back( + dev_name + ":" + it->second.as()); + } + + // Configure devices fallback order for proxy_plugin + // Can use substring to configure the order + // CUDA iGPU : CUDA iGPU // just create a new elememnt + // CPU iGPU : CUDA CPU iGPU // use substring to find the right place + it = config.find(ov::proxy::configuration::fallback.name()); + if (it != config.end()) { + auto fallback = it->second.as(); + // Change fallback name if fallback is configured to the HW plugin under the proxy with the same name + if (alias == fallback) + fallback += internal_plugin_suffix; + if (defaultConfig.find(ov::device::priorities.name()) == defaultConfig.end()) { + defaultConfig[ov::device::priorities.name()] = std::vector{dev_name, fallback}; + } else { + auto dev_order = defaultConfig[ov::device::priorities.name()].as>(); + auto begin_it = std::find(dev_order.begin(), dev_order.end(), dev_name); + auto end_it = std::find(dev_order.begin(), dev_order.end(), fallback); + OPENVINO_ASSERT(begin_it == dev_order.end() && end_it == dev_order.end(), + "Cannot restore the fallback order for proxy plugin."); + if (begin_it != dev_order.end() && end_it != dev_order.end()) { + // Nothing to do. Just check that devices have the right order + OPENVINO_ASSERT(std::distance(begin_it, end_it) > 0, + "Incorrect order of proxy plugin fallback priority."); + } else if (begin_it != dev_order.end()) { + // Insert fallback device after the primary device + dev_order.insert(begin_it + 1, fallback); + } else if (end_it != dev_order.end()) { + // Insert primary device before the fallback device + dev_order.insert(end_it, dev_name); + } + defaultConfig[ov::device::priorities.name()] = dev_order; + } + } + }; +#endif + + auto&& config = desc.defaultConfig; + std::string dev_name = device_name; + // Register proxy plugin + if (config.find(ov::proxy::configuration::alias.name()) != config.end()) { +#ifndef PROXY_PLUGIN_ENABLED + OPENVINO_THROW("Cannot register plugin under the proxy. Proxy plugin is disabled."); +#else + // Create proxy plugin for alias + auto alias = config.at(ov::proxy::configuration::alias.name()).as(); + if (alias == device_name) + dev_name += internal_plugin_suffix; + // Alias can be registered by several plugins + if (pluginRegistry.find(alias) == pluginRegistry.end()) { + // Register new plugin + PluginDescriptor desc = PluginDescriptor(ov::proxy::create_plugin); + fill_config(desc.defaultConfig, config, dev_name); + pluginRegistry[alias] = desc; + add_mutex(alias); + } else { + // Update registered plugin + auto& plugin = pluginRegistry.at(alias); + // Error if we have an alias for HW plugin + OPENVINO_ASSERT(plugin.pluginCreateFunc == ov::proxy::create_plugin, + "Cannot register plugin for ", + dev_name, + " plugin with the same name already registered!"); + fill_config(plugin.defaultConfig, config, dev_name); + } +#endif + } else if (config.find(ov::proxy::configuration::fallback.name()) != config.end()) { +#ifndef PROXY_PLUGIN_ENABLED + OPENVINO_THROW("Cannot register plugin under the proxy. Proxy plugin is disabled."); +#else + // Fallback without alias means that we need to replace original plugin to proxy + dev_name += internal_plugin_suffix; + PluginDescriptor desc = PluginDescriptor(ov::proxy::create_plugin); + fill_config(desc.defaultConfig, config, dev_name); + pluginRegistry[device_name] = desc; + add_mutex(device_name); +#endif + } + + const static std::vector proxy_conf_properties = {ov::proxy::configuration::alias, + ov::proxy::configuration::fallback, + ov::proxy::configuration::priority}; + + // Register real plugin + for (const auto& proxy_prop : proxy_conf_properties) { + auto it = desc.defaultConfig.find(proxy_prop); + if (it != desc.defaultConfig.end()) { + desc.defaultConfig.erase(it); + } + } + + pluginRegistry[dev_name] = desc; + add_mutex(dev_name); +} + void ov::CoreImpl::register_compile_time_plugins() { std::lock_guard lock(get_mutex()); @@ -323,16 +459,14 @@ void ov::CoreImpl::register_compile_time_plugins() { const auto& value = plugin.second; ov::AnyMap config = any_copy(value.m_default_config); PluginDescriptor desc{value.m_create_plugin_func, config, value.m_create_extension_func}; - pluginRegistry[deviceName] = desc; - add_mutex(deviceName); + register_plugin_in_registry_unsafe(deviceName, desc); } #else const auto& pluginPath = ov::util::get_compiled_plugin_path(plugin.second.m_plugin_path); if (pluginRegistry.find(deviceName) == pluginRegistry.end() && ov::util::file_exists(pluginPath)) { ov::AnyMap config = any_copy(plugin.second.m_default_config); PluginDescriptor desc{pluginPath, config}; - pluginRegistry[deviceName] = desc; - add_mutex(deviceName); + register_plugin_in_registry_unsafe(deviceName, desc); } #endif } @@ -391,8 +525,7 @@ void ov::CoreImpl::register_plugins_in_registry(const std::string& xml_config_fi // fill value in plugin registry for later lazy initialization { PluginDescriptor desc{pluginPath, config, listOfExtentions}; - pluginRegistry[deviceName] = desc; - add_mutex(deviceName); + register_plugin_in_registry_unsafe(deviceName, desc); } } } @@ -437,7 +570,7 @@ ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName) const { try { ov::Plugin plugin; - if (desc.pluginCreateFunc) { // static OpenVINO case + if (desc.pluginCreateFunc) { // static OpenVINO case or proxy plugin std::shared_ptr plugin_impl; desc.pluginCreateFunc(plugin_impl); plugin = Plugin{plugin_impl, {}}; @@ -466,6 +599,26 @@ ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName) const { // configuring { +#ifdef PROXY_PLUGIN_ENABLED + // Initial setup for proxy plugin. + // It is needed for future initialization to initialize low level plugin + if (desc.pluginCreateFunc == ov::proxy::create_plugin) { + ov::AnyMap initial_config; + auto it = desc.defaultConfig.find(ov::proxy::alias_for.name()); + if (it != desc.defaultConfig.end()) { + initial_config[it->first] = it->second; + } + it = desc.defaultConfig.find(ov::proxy::device_priorities.name()); + if (it != desc.defaultConfig.end()) { + initial_config[it->first] = it->second; + } + it = desc.defaultConfig.find(ov::device::priorities.name()); + if (it != desc.defaultConfig.end()) { + initial_config[ov::device::priorities.name()] = it->second; + } + plugin.set_property(initial_config); + } +#endif // TODO: remove this block of code once GPU removes support of ov::cache_dir // also, remove device_supports_cache_dir at all { @@ -556,7 +709,10 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::shared_ptr< if (cacheManager && device_supports_model_caching(plugin)) { CacheContent cacheContent{cacheManager}; cacheContent.blobId = ov::ModelCache::compute_hash(model, create_compile_config(plugin, parsed._config)); - auto lock = cacheGuard.get_hash_lock(cacheContent.blobId); + std::unique_ptr lock; + // Proxy plugin fallback to lowlevel device + if (!is_proxy_device(plugin)) + lock = cacheGuard.get_hash_lock(cacheContent.blobId); res = load_model_from_cache(cacheContent, plugin, parsed._config, ov::RemoteContext{}, [&]() { return compile_model_and_cache(model, plugin, parsed._config, ov::RemoteContext{}, cacheContent); }); @@ -585,7 +741,10 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::shared_ptr< if (cacheManager && device_supports_model_caching(plugin)) { CacheContent cacheContent{cacheManager}; cacheContent.blobId = ov::ModelCache::compute_hash(model, create_compile_config(plugin, parsed._config)); - auto lock = cacheGuard.get_hash_lock(cacheContent.blobId); + std::unique_ptr lock; + // Proxy plugin fallback to lowlevel device + if (!is_proxy_device(plugin)) + lock = cacheGuard.get_hash_lock(cacheContent.blobId); res = load_model_from_cache(cacheContent, plugin, parsed._config, context, [&]() { return compile_model_and_cache(model, plugin, parsed._config, context, cacheContent); }); @@ -601,8 +760,9 @@ ov::SoPtr ov::CoreImpl::compile_model_with_preprocess(ov::Pl const ov::AnyMap& config) const { std::shared_ptr preprocessed_model = model; + // Disable conversion for proxy plugin and virtual devices to add pre-processing based on API of internal plugins if (!is_new_api() && !std::dynamic_pointer_cast(plugin.m_ptr) && - !is_virtual_device(plugin.get_name())) { + !is_virtual_device(plugin.get_name()) && !is_proxy_device(plugin)) { ov::pass::Manager manager; manager.register_pass(); @@ -628,7 +788,10 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::string& mod if (cacheManager && device_supports_model_caching(plugin)) { CacheContent cacheContent{cacheManager, model_path}; cacheContent.blobId = ov::ModelCache::compute_hash(model_path, create_compile_config(plugin, parsed._config)); - auto lock = cacheGuard.get_hash_lock(cacheContent.blobId); + std::unique_ptr lock; + // Proxy plugin fallback to lowlevel device + if (!is_proxy_device(plugin)) + lock = cacheGuard.get_hash_lock(cacheContent.blobId); compiled_model = load_model_from_cache(cacheContent, plugin, parsed._config, ov::RemoteContext{}, [&]() { auto cnnNetwork = ReadNetwork(model_path, std::string()); return compile_model_and_cache(cnnNetwork.getFunction(), plugin, parsed._config, {}, cacheContent); @@ -660,7 +823,10 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::string& mod CacheContent cacheContent{cacheManager}; cacheContent.blobId = ov::ModelCache::compute_hash(model_str, weights, create_compile_config(plugin, parsed._config)); - auto lock = cacheGuard.get_hash_lock(cacheContent.blobId); + std::unique_ptr lock; + // Proxy plugin fallback to lowlevel device + if (!is_proxy_device(plugin)) + lock = cacheGuard.get_hash_lock(cacheContent.blobId); compiled_model = load_model_from_cache(cacheContent, plugin, parsed._config, ov::RemoteContext{}, [&]() { auto cnnNetwork = read_model(model_str, weights); return compile_model_and_cache(cnnNetwork, plugin, parsed._config, ov::RemoteContext{}, cacheContent); @@ -706,12 +872,36 @@ ov::SupportedOpsMap ov::CoreImpl::query_model(const std::shared_ptr lock(get_mutex()); + if (device_name.find(internal_plugin_suffix) != std::string::npos) + return true; + + // Alias hides the device + for (auto&& it : pluginRegistry) { + auto it_priority = it.second.defaultConfig.find(ov::proxy::alias_for.name()); + if (it.first == device_name || it_priority == it.second.defaultConfig.end()) + continue; + auto devices = it_priority->second.as>(); + for (const auto& dev : devices) { + if (dev == device_name) + return true; + } + } +#endif + return false; +} + std::vector ov::CoreImpl::get_available_devices() const { std::vector devices; const std::string propertyName = METRIC_KEY(AVAILABLE_DEVICES); for (auto&& deviceName : get_registered_devices()) { std::vector devicesIDs; + // Skip hidden devices + if (is_hidden_device(deviceName)) + continue; try { const ie::Parameter p = GetMetric(deviceName, propertyName); devicesIDs = p.as>(); @@ -863,6 +1053,9 @@ std::shared_ptr ov::CoreImpl::apply_auto_batching(const std::sh // check whether if the Auto-Batching is applicable to the device auto parsed = ov::parseDeviceNameIntoConfig(deviceName); + // Do not apply auto batch for proxy device + if (is_proxy_device(parsed._deviceName)) + return model; deviceNameWithoutBatch = deviceName; std::vector metrics = get_plugin(parsed._deviceName) .get_property(METRIC_KEY(SUPPORTED_METRICS), parsed._config) @@ -915,7 +1108,9 @@ void ov::CoreImpl::set_property(const std::string& device_name, const AnyMap& pr auto devices = get_registered_devices(); for (auto&& config : properties) { const auto is_secondary_property = config.first.find(ov::device::properties.name()) != std::string::npos; - OPENVINO_ASSERT(!is_secondary_property, + // It is valid change for proxy plugin, proxy plugin allows to set properties for low level fallback devices + const auto is_proxy = is_proxy_device(ov::parseDeviceNameIntoConfig(device_name)._deviceName); + OPENVINO_ASSERT(!is_secondary_property || is_proxy, "set_property do not support ov::device::propreties. " "You can configure the devices through the compile_model()/query_model() API."); } @@ -974,7 +1169,9 @@ void ov::CoreImpl::unload_plugin(const std::string& deviceName) { plugins.erase(deviceName); } -void ov::CoreImpl::register_plugin(const std::string& plugin, const std::string& device_name) { +void ov::CoreImpl::register_plugin(const std::string& plugin, + const std::string& device_name, + const ov::AnyMap& properties) { std::lock_guard lock(get_mutex()); auto it = pluginRegistry.find(device_name); @@ -986,9 +1183,8 @@ void ov::CoreImpl::register_plugin(const std::string& plugin, const std::string& IE_THROW() << "Device name must not contain dot '.' symbol"; } - PluginDescriptor desc{ov::util::get_plugin_path(plugin)}; - pluginRegistry[device_name] = desc; - add_mutex(device_name); + PluginDescriptor desc{ov::util::get_plugin_path(plugin), properties}; + register_plugin_in_registry_unsafe(device_name, desc); } /** diff --git a/src/inference/src/dev/core_impl.hpp b/src/inference/src/dev/core_impl.hpp index 84fafa329e2..0f132fe1af5 100644 --- a/src/inference/src/dev/core_impl.hpp +++ b/src/inference/src/dev/core_impl.hpp @@ -66,6 +66,9 @@ private: std::mutex& get_mutex(const std::string& dev_name = "") const; void add_mutex(const std::string& dev_name); + bool is_proxy_device(const ov::Plugin& plugin) const; + bool is_proxy_device(const std::string& dev_name) const; + class CoreConfig final { public: struct CacheConfig { @@ -179,6 +182,9 @@ private: ov::AnyMap create_compile_config(const ov::Plugin& plugin, const ov::AnyMap& origConfig) const; + bool is_hidden_device(const std::string& device_name) const; + void register_plugin_in_registry_unsafe(const std::string& device_name, PluginDescriptor& desc); + template > void try_to_register_plugin_extensions(const std::basic_string& path) const { try { @@ -335,8 +341,9 @@ public: * @param plugin Path (absolute or relative) or name of a plugin. Depending on platform `plugin` is wrapped with * shared library suffix and prefix to identify library full name * @param device_name A name of device + * @param properties Plugin configuration */ - void register_plugin(const std::string& plugin, const std::string& device_name); + void register_plugin(const std::string& plugin, const std::string& device_name, const ov::AnyMap& properties); /** * @brief Provides a list of plugin names in registry; physically such plugins may not be created @@ -410,7 +417,7 @@ public: * * @param properties Map of pairs: (property name, property value). */ - void set_property(const std::string& device_name, const AnyMap& properties); + void set_property(const std::string& device_name, const AnyMap& properties) override; /** * @brief Sets properties for a device, acceptable keys can be found in openvino/runtime/properties.hpp. diff --git a/src/inference/src/dev/icompiled_model_wrapper.cpp b/src/inference/src/dev/icompiled_model_wrapper.cpp index 544eab2c4a3..354ca304fd2 100644 --- a/src/inference/src/dev/icompiled_model_wrapper.cpp +++ b/src/inference/src/dev/icompiled_model_wrapper.cpp @@ -23,7 +23,7 @@ InferenceEngine::ICompiledModelWrapper::ICompiledModelWrapper( } std::shared_ptr InferenceEngine::ICompiledModelWrapper::create_infer_request() const { - return ov::legacy_convert::convert_infer_request(m_model->CreateInferRequest()); + return ov::legacy_convert::convert_infer_request(m_model->CreateInferRequest(), m_model->_plugin->GetName()); } void InferenceEngine::ICompiledModelWrapper::export_model(std::ostream& model) const { diff --git a/src/inference/src/dev/make_tensor.cpp b/src/inference/src/dev/make_tensor.cpp index b96548df7b5..05101d82782 100644 --- a/src/inference/src/dev/make_tensor.cpp +++ b/src/inference/src/dev/make_tensor.cpp @@ -12,6 +12,9 @@ #include "ie_remote_blob.hpp" #include "openvino/runtime/iremote_tensor.hpp" #include "openvino/runtime/properties.hpp" +#ifdef PROXY_PLUGIN_ENABLED +# include "openvino/proxy/plugin.hpp" +#endif namespace ov { @@ -583,7 +586,42 @@ std::shared_ptr make_tensor(const std::shared_ptr& blob) { #undef IF } -ie::Blob::Ptr tensor_to_blob(const std::shared_ptr& tensor) { +ie::Blob* get_hardware_blob(ie::Blob* blob) { +#ifdef PROXY_PLUGIN_ENABLED + if (auto remote_blob = dynamic_cast(blob)) { + const auto& tensor = ov::proxy::get_hardware_tensor(remote_blob->tensor); + if (auto blob_tensor = std::dynamic_pointer_cast(tensor)) { + return blob_tensor->blob.get(); + } else if (auto blob_tensor = std::dynamic_pointer_cast(tensor)) { + return blob_tensor->blob.get(); + } + OPENVINO_NOT_IMPLEMENTED; + } +#endif + return blob; +} + +const ie::Blob* get_hardware_blob(const ie::Blob* blob) { +#ifdef PROXY_PLUGIN_ENABLED + if (auto remote_blob = dynamic_cast(blob)) { + const auto& tensor = ov::proxy::get_hardware_tensor(remote_blob->tensor); + if (auto blob_tensor = std::dynamic_pointer_cast(tensor)) { + return blob_tensor->blob.get(); + } else if (auto blob_tensor = std::dynamic_pointer_cast(tensor)) { + return blob_tensor->blob.get(); + } + OPENVINO_NOT_IMPLEMENTED; + } +#endif + return blob; +} + +ie::Blob::Ptr tensor_to_blob(const std::shared_ptr& orig_tensor, bool unwrap) { +#ifdef PROXY_PLUGIN_ENABLED + const auto& tensor = unwrap ? ov::proxy::get_hardware_tensor(orig_tensor) : orig_tensor; +#else + const auto& tensor = orig_tensor; +#endif if (tensor == nullptr) { return {}; } else if (auto blob_tensor = std::dynamic_pointer_cast(tensor)) { diff --git a/src/inference/src/dev/make_tensor.hpp b/src/inference/src/dev/make_tensor.hpp index 88c338a096b..10f35ea66c8 100644 --- a/src/inference/src/dev/make_tensor.hpp +++ b/src/inference/src/dev/make_tensor.hpp @@ -46,8 +46,10 @@ std::shared_ptr make_tensor(const std::shared_ptr& other, IE_SUPPRESS_DEPRECATED_START /** @cond INTERNAL */ std::shared_ptr make_tensor(const std::shared_ptr& tensor); +const InferenceEngine::Blob* get_hardware_blob(const InferenceEngine::Blob* blob); +InferenceEngine::Blob* get_hardware_blob(InferenceEngine::Blob* blob); -std::shared_ptr tensor_to_blob(const std::shared_ptr& tensor); +std::shared_ptr tensor_to_blob(const std::shared_ptr& tensor, bool unwrap = true); /** @endcond */ IE_SUPPRESS_DEPRECATED_END diff --git a/src/inference/src/dev/remote_context_wrapper.hpp b/src/inference/src/dev/remote_context_wrapper.hpp new file mode 100644 index 00000000000..78be59eeed7 --- /dev/null +++ b/src/inference/src/dev/remote_context_wrapper.hpp @@ -0,0 +1,50 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include "dev/make_tensor.hpp" +#include "ie_ngraph_utils.hpp" +#include "ie_remote_context.hpp" +#include "openvino/runtime/iremote_context.hpp" + +namespace ov { + +class RemoteContextWrapper : public InferenceEngine::RemoteContext { +private: + std::shared_ptr m_context; + +public: + RemoteContextWrapper(const std::shared_ptr& context) : m_context(context) {} + + const std::shared_ptr& get_context() const { + return m_context; + } + + std::string getDeviceName() const noexcept override { + return m_context->get_device_name(); + } + + InferenceEngine::RemoteBlob::Ptr CreateBlob(const InferenceEngine::TensorDesc& tensorDesc, + const InferenceEngine::ParamMap& params = {}) override { + return std::dynamic_pointer_cast(ov::tensor_to_blob( + m_context->create_tensor(InferenceEngine::details::convertPrecision(tensorDesc.getPrecision()), + tensorDesc.getBlockingDesc().getBlockDims(), + params), + false)); + } + + InferenceEngine::MemoryBlob::Ptr CreateHostBlob(const InferenceEngine::TensorDesc& tensorDesc) override { + return std::dynamic_pointer_cast(ov::tensor_to_blob( + m_context->create_host_tensor(InferenceEngine::details::convertPrecision(tensorDesc.getPrecision()), + tensorDesc.getBlockingDesc().getBlockDims()), + false)); + } + + InferenceEngine::ParamMap getParams() const override { + return m_context->get_property(); + } +}; + +} // namespace ov diff --git a/src/inference/src/ie_blob_common.cpp b/src/inference/src/ie_blob_common.cpp index 9055445a573..c5d496bbf57 100644 --- a/src/inference/src/ie_blob_common.cpp +++ b/src/inference/src/ie_blob_common.cpp @@ -6,12 +6,29 @@ #include #include +#include "dev/make_tensor.hpp" #include "ie_blob.h" #include "system_allocator.hpp" namespace InferenceEngine { IE_SUPPRESS_DEPRECATED_START +Blob* Blob::getHardwareBlob() { +#ifdef PROXY_PLUGIN_ENABLED + return ov::get_hardware_blob(this); +#else + return this; +#endif +} + +const Blob* Blob::getHardwareBlob() const { +#ifdef PROXY_PLUGIN_ENABLED + return ov::get_hardware_blob(this); +#else + return this; +#endif +} + void Blob::setShape(const SizeVector& dims) { // we don't want to allow setShape for: // 1. ROI cases diff --git a/src/inference/src/ie_core.cpp b/src/inference/src/ie_core.cpp index 22327acc335..ae393f31ea0 100644 --- a/src/inference/src/ie_core.cpp +++ b/src/inference/src/ie_core.cpp @@ -338,7 +338,7 @@ std::vector Core::GetAvailableDevices() const { } void Core::RegisterPlugin(const std::string& pluginName, const std::string& deviceName) { - _impl->register_plugin(pluginName, deviceName); + _impl->register_plugin(pluginName, deviceName, {}); } void Core::RegisterPlugins(const std::string& xmlConfigFile) { diff --git a/src/inference/src/ie_remote_context.cpp b/src/inference/src/ie_remote_context.cpp index 9aef0f76afc..9430a05e71b 100644 --- a/src/inference/src/ie_remote_context.cpp +++ b/src/inference/src/ie_remote_context.cpp @@ -8,6 +8,12 @@ #include #include "blob_factory.hpp" +#include "dev/converter_utils.hpp" +#include "dev/remote_context_wrapper.hpp" +#include "openvino/runtime/remote_context.hpp" +#ifdef PROXY_PLUGIN_ENABLED +# include "openvino/proxy/plugin.hpp" +#endif namespace InferenceEngine { @@ -19,6 +25,28 @@ MemoryBlob::Ptr RemoteContext::CreateHostBlob(const TensorDesc& tensorDesc) { return blob; } + +const std::shared_ptr RemoteContext::GetHardwareContext() { +#ifdef PROXY_PLUGIN_ENABLED + if (auto wrapper = dynamic_cast(this)) { + auto ov_context = wrapper->get_context(); + auto hw_context = ov::proxy::get_hardware_context(ov_context); + return ov::legacy_convert::convert_remote_context(hw_context); + } +#endif + return shared_from_this(); +} + +const std::shared_ptr RemoteContext::GetHardwareContext() const { +#ifdef PROXY_PLUGIN_ENABLED + if (auto wrapper = dynamic_cast(this)) { + auto ov_context = wrapper->get_context(); + auto hw_context = ov::proxy::get_hardware_context(ov_context); + return ov::legacy_convert::convert_remote_context(hw_context); + } +#endif + return shared_from_this(); +} IE_SUPPRESS_DEPRECATED_END } // namespace InferenceEngine diff --git a/src/plugins/auto_batch/src/compiled_model.cpp b/src/plugins/auto_batch/src/compiled_model.cpp index 72354f4d0e9..6ec9d99d3ee 100644 --- a/src/plugins/auto_batch/src/compiled_model.cpp +++ b/src/plugins/auto_batch/src/compiled_model.cpp @@ -242,5 +242,6 @@ InferenceEngine::Parameter CompiledModel::GetMetric(const std::string& name) con IE_THROW() << "Unsupported Network metric: " << name; } } + } // namespace autobatch_plugin -} // namespace ov \ No newline at end of file +} // namespace ov diff --git a/src/plugins/auto_batch/src/compiled_model.hpp b/src/plugins/auto_batch/src/compiled_model.hpp index de970dcf36a..8259143b8eb 100644 --- a/src/plugins/auto_batch/src/compiled_model.hpp +++ b/src/plugins/auto_batch/src/compiled_model.hpp @@ -80,4 +80,4 @@ protected: const std::set m_batched_outputs; }; } // namespace autobatch_plugin -} // namespace ov \ No newline at end of file +} // namespace ov diff --git a/src/plugins/intel_gpu/CMakeLists.txt b/src/plugins/intel_gpu/CMakeLists.txt index 948aecfcfec..ffb42d981c3 100644 --- a/src/plugins/intel_gpu/CMakeLists.txt +++ b/src/plugins/intel_gpu/CMakeLists.txt @@ -46,9 +46,14 @@ add_subdirectory(src/graph) file(GLOB_RECURSE PLUGIN_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/plugin/*.cpp ${CMAKE_CURRENT_SOURCE_DIR}/include/intel_gpu/plugin/*.hpp) +if(ENABLE_PROXY) + set(PLUGIN_DEFAULT_CONFIG "PROXY_CONFIGURATION_ALIAS:GPU;PROXY_CONFIGURATION_PRIORITY:0") +endif() + ov_add_plugin(NAME ${TARGET_NAME} DEVICE_NAME "GPU" SOURCES ${PLUGIN_SOURCES} + DEFAULT_CONFIG ${PLUGIN_DEFAULT_CONFIG} VERSION_DEFINES_FOR src/plugin/plugin.cpp) target_compile_options(${TARGET_NAME} PRIVATE diff --git a/src/plugins/intel_gpu/src/graph/primitive_inst.cpp b/src/plugins/intel_gpu/src/graph/primitive_inst.cpp index 6d5dea9790b..16f3a9fe1a5 100644 --- a/src/plugins/intel_gpu/src/graph/primitive_inst.cpp +++ b/src/plugins/intel_gpu/src/graph/primitive_inst.cpp @@ -153,7 +153,7 @@ void primitive_inst::check_memory_to_set(const memory& mem, const layout& layout OPENVINO_ASSERT(mem.is_allocated_by(net_engine), "[GPU] Can't set memory due to engines mismatch. ", "Network was created for ", &net_engine, " (", net_engine.get_device_info().dev_name, ") engine", - " while memory object was allocated for ", &mem_engine, "(", + " while memory object was allocated for ", &mem_engine, " (", mem_engine.get_device_info().dev_name, ")"); switch (params.mem_type) { diff --git a/src/plugins/intel_gpu/tests/functional/CMakeLists.txt b/src/plugins/intel_gpu/tests/functional/CMakeLists.txt index 6ebf7dbbccb..fb274376a9c 100644 --- a/src/plugins/intel_gpu/tests/functional/CMakeLists.txt +++ b/src/plugins/intel_gpu/tests/functional/CMakeLists.txt @@ -31,6 +31,10 @@ addIeTargetTest( GPU ) +if(ENABLE_PROXY) + target_compile_definitions(${TARGET_NAME} PUBLIC PROXY_PLUGIN_ENABLED) +endif() + # try to find VA libraries if(NOT ANDROID) find_package(PkgConfig QUIET) diff --git a/src/plugins/intel_gpu/tests/functional/remote_blob_tests/cldnn_remote_blob_tests.cpp b/src/plugins/intel_gpu/tests/functional/remote_blob_tests/cldnn_remote_blob_tests.cpp index 04fbb6d7419..7516b301dca 100644 --- a/src/plugins/intel_gpu/tests/functional/remote_blob_tests/cldnn_remote_blob_tests.cpp +++ b/src/plugins/intel_gpu/tests/functional/remote_blob_tests/cldnn_remote_blob_tests.cpp @@ -74,7 +74,7 @@ TEST_P(RemoteBlob_Test, smoke_canInputUserBlob) { // inference using remote blob auto inf_req_shared = exec_net.CreateInferRequest(); auto cldnn_context = exec_net.GetContext(); - cl_context ctx = std::dynamic_pointer_cast(cldnn_context)->get(); + cl_context ctx = cldnn_context->as()->get(); auto ocl_instance = std::make_shared(ctx); cl_int err; @@ -190,7 +190,7 @@ TEST_P(RemoteBlob_Test, smoke_canInputPluginRemoteBlob) { // inference using remote blob auto inf_req_shared = exec_net.CreateInferRequest(); auto cldnn_context = exec_net.GetContext(); - cl_context ctx = std::dynamic_pointer_cast(cldnn_context)->get(); + cl_context ctx = cldnn_context->as()->get(); auto ocl_instance = std::make_shared(ctx); auto desc = net.getInputsInfo().begin()->second->getTensorDesc(); diff --git a/src/plugins/intel_gpu/tests/functional/shared_tests_instances/skip_tests_config.cpp b/src/plugins/intel_gpu/tests/functional/shared_tests_instances/skip_tests_config.cpp index 89843c8cf09..97fa61f995b 100644 --- a/src/plugins/intel_gpu/tests/functional/shared_tests_instances/skip_tests_config.cpp +++ b/src/plugins/intel_gpu/tests/functional/shared_tests_instances/skip_tests_config.cpp @@ -137,5 +137,11 @@ std::vector disabledTestPatterns() { // For some strange reason (bug?) output format cannot have a rank greater than 4 for dynamic shape case, // because it crashes in some random places during "reorder_inputs" pass. R"(.*UniqueLayerDynamicGPUTest.*\(\d*\.\d*\.\d*\.\d*\.\d*\).*axis.*)", +#ifdef PROXY_PLUGIN_ENABLED + // Plugin version was changed to ov::Version + R"(.*VersionTest.*pluginCurrentVersionIsCorrect.*)", + // Issue because meta plugins use OLD API and wrap proxy which work with new API + R"(.*OVHoldersTest.*LoadedTensor.*HETERO.*)", +#endif }; } diff --git a/src/plugins/proxy/CMakeLists.txt b/src/plugins/proxy/CMakeLists.txt new file mode 100644 index 00000000000..4f5b989aef4 --- /dev/null +++ b/src/plugins/proxy/CMakeLists.txt @@ -0,0 +1,49 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +if (NOT ENABLE_PROXY) + return() +endif() + + +set(TARGET_NAME "openvino_proxy_plugin_obj") + +set(PUBLIC_HEADERS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/dev_api) +file(GLOB LIBRARY_SRC ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp) +file(GLOB LIBRARY_HEADERS ${PUBLIC_HEADERS_DIR}/*.hpp ${CMAKE_CURRENT_SOURCE_DIR}/src/*.hpp) + +# Create named folders for the sources within the .vcproj +# Empty name lists them directly under the .vcproj +source_group("src" FILES ${LIBRARY_SRC}) +source_group("include" FILES ${LIBRARY_HEADERS}) + +add_library(${TARGET_NAME} OBJECT ${LIBRARY_SRC} ${LIBRARY_HEADERS}) + +if(NOT BUILD_SHARED_LIBS) + target_compile_definitions(${TARGET_NAME} PUBLIC OPENVINO_STATIC_LIBRARY) +endif() + +target_compile_definitions(${TARGET_NAME} PRIVATE + IMPLEMENT_INFERENCE_ENGINE_API + CI_BUILD_NUMBER=\"${CI_BUILD_NUMBER}\") + +# LTO +set_target_properties(${TARGET_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO}) + +target_include_directories(${TARGET_NAME} PUBLIC $ + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) + +target_include_directories(${TARGET_NAME} SYSTEM PRIVATE + $>) + +add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME}) + +ov_mark_target_as_cc(${TARGET_NAME}) + +ov_ncc_naming_style(FOR_TARGET ${TARGET_NAME} + SOURCE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/dev_api" + "${CMAKE_CURRENT_SOURCE_DIR}/src") + +if(ENABLE_TESTS) + add_subdirectory(tests) +endif() diff --git a/src/plugins/proxy/README.md b/src/plugins/proxy/README.md new file mode 100644 index 00000000000..62ea9c7ae22 --- /dev/null +++ b/src/plugins/proxy/README.md @@ -0,0 +1,50 @@ +# OpenVINO Proxy Plugin + +## Key Contacts + +Please contact a member of [openvino-ie-maintainers](https://github.com/orgs/openvinotoolkit/teams/openvino-ie-maintainers) group, for assistance regarding Proxy plugin. + +## Components + +Proxy Plugin contains the following components: + +* [include](./include/) - folder contains public plugin API. +* [src](./src/) - folder contains sources of the plugin. +* [tests](./tests/) - contains tests for the plugin. + +## Motivation + + - OpenVINO may have multiple hardware plugins for similar device type from different vendors (e.g. Intel and NVidia GPUs) and currently user must address them with different names ("GPU" and "NVIDIA" respectively). Using same name for such cases seems to be more user-friendly approach. + - Moreover, single physical device may have multiple plugin which support it. For example, Intel GPU plugin is OpenCL based, thus can run on other OCL-compatible devices including NVIDIA gpus. In that case we may have primary plugin ("NVIDIA") which provides best performance on target device and fallback plugin ("INTEL_GPU") which helps to improve models coverage for the cases when primary plugin has limited operation set support, and both plugins may be executed via HETERO mode. + - Implicit HETERO plugin usage may be extended to run on different device types - HETERO:xPU,CPU + +## Requirements + + - Do not provide additional libraries and don't affect load time (proxy plugin is a part of openvino library) + - No overhead for load inference time + - Fallback to hardware plugin if device is supported only by one plugin + - Minimal overhead in case of multiple plugins for one device + - Plus one call of query network if entire model can be executed on preferable plugin + - Hetero execution in other case + - Allow to configure device + +## Proxy plugin properties + +Proxy plugin cannot be created explicitly. In order to use proxy plugin under the real hardware plugins please use next properties for configuration: + - `ov::proxy::configuration::alias` is an alias name for high level plugin. + - `ov::proxy::configuration::priority` is a device priority under alias (lower value means the higher priority), this value allows to configure the device order. + - `ov::proxy::configuration::fallback` the name of other hardware device for the fallback. + +After the creation the proxy plugin has next properties: + - `ov::device::priorities` is fallback order inside the proxy plugin. + - `ov::proxy::device_priorities` is a device order under proxy plugin. + - `ov::proxy::alias_for` the vector of low level plugins under the proxy alias. + + +## See also + * [OpenVINO™ README](../../../README.md) + * [OpenVINO Core Components](../../README.md) + * [OpenVINO Plugins](../README.md) + * [Developer documentation](../../../docs/dev/index.md) + * [OpenVINO Plugin Developer Guide](https://docs.openvino.ai/latest/openvino_docs_ie_plugin_dg_overview.html) + diff --git a/src/plugins/proxy/dev_api/openvino/proxy/infer_request.hpp b/src/plugins/proxy/dev_api/openvino/proxy/infer_request.hpp new file mode 100644 index 00000000000..88aec1dd0f8 --- /dev/null +++ b/src/plugins/proxy/dev_api/openvino/proxy/infer_request.hpp @@ -0,0 +1,57 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#pragma once + +#include + +#include "openvino/runtime/iasync_infer_request.hpp" +#include "openvino/runtime/icompiled_model.hpp" +#include "openvino/runtime/so_ptr.hpp" + +namespace ov { +namespace proxy { + +class InferRequest : public ov::IAsyncInferRequest { +private: + ov::SoPtr m_infer_request; + std::shared_ptr m_compiled_model; + +public: + InferRequest(ov::SoPtr&& request, + const std::shared_ptr& compiled_model); + void start_async() override; + + void wait() override; + + bool wait_for(const std::chrono::milliseconds& timeout) override; + + void cancel() override; + + void set_callback(std::function callback) override; + + void infer() override; + + std::vector get_profiling_info() const override; + + ov::Tensor get_tensor(const ov::Output& port) const override; + + void set_tensor(const ov::Output& port, const ov::Tensor& tensor) override; + + std::vector get_tensors(const ov::Output& port) const override; + + void set_tensors(const ov::Output& port, const std::vector& tensors) override; + + std::vector> query_state() const override; + + const std::shared_ptr& get_compiled_model() const override; + + const std::vector>& get_inputs() const override; + + const std::vector>& get_outputs() const override; + + const ov::SoPtr get_hardware_request() const; +}; + +} // namespace proxy +} // namespace ov diff --git a/src/plugins/proxy/dev_api/openvino/proxy/plugin.hpp b/src/plugins/proxy/dev_api/openvino/proxy/plugin.hpp new file mode 100644 index 00000000000..e69d320ca21 --- /dev/null +++ b/src/plugins/proxy/dev_api/openvino/proxy/plugin.hpp @@ -0,0 +1,41 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include "openvino/runtime/iplugin.hpp" +#include "openvino/runtime/iremote_context.hpp" + +namespace ov { +namespace proxy { + +/** + * @brief Creates a new instance of Proxy plugin + * + * @param plugin shared pointer to the plugin + */ +void create_plugin(std::shared_ptr& plugin); + +/** + * @brief Get wrapped remote context + * + * @param context Remote context + * + * @return Original remote context + */ +const std::shared_ptr& get_hardware_context(const std::shared_ptr& context); + +/** + * @brief Get wrapped remote tensor + * + * @param tensor Remote tensor + * + * @return Original remote tensor + */ +const std::shared_ptr& get_hardware_tensor(const std::shared_ptr& tensor); + +} // namespace proxy +} // namespace ov diff --git a/src/plugins/proxy/dev_api/openvino/proxy/properties.hpp b/src/plugins/proxy/dev_api/openvino/proxy/properties.hpp new file mode 100644 index 00000000000..74568e71c74 --- /dev/null +++ b/src/plugins/proxy/dev_api/openvino/proxy/properties.hpp @@ -0,0 +1,54 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include "openvino/runtime/properties.hpp" + +namespace ov { +namespace proxy { + +// Proxy plugin configuration properties +namespace configuration { +/** + * @brief Read-write property to set alias for hardware plugin + * value type: string Alias name for the set of plugins + * @ingroup ov_runtime_cpp_prop_api + */ +static constexpr Property alias{"PROXY_CONFIGURATION_ALIAS"}; + +/** + * @brief Read-write property to set devices priority in alias + * This property allows to configure the order of devices from different low-level plugins under the proxy + * value type: int32_t lower value means the higher priority + * @ingroup ov_runtime_cpp_prop_api + */ +static constexpr Property priority{"PROXY_CONFIGURATION_PRIORITY"}; + +/** + * @brief Read-write property to set the fallback to other HW plugin + * value type: string the name of hardware plugin for fallback + * @ingroup ov_runtime_cpp_prop_api + */ +static constexpr Property fallback{"PROXY_CONFIGURATION_FALLBACK"}; + +} // namespace configuration + +/** + * @brief Property allows to configure the low level device priorities. + * Vector of string. String has the next format: : + */ +static constexpr Property, PropertyMutability::RW> device_priorities{ + "PROXY_DEVICE_PRIORITIES"}; + +/** + * @brief Property allows to configure the list of low level devices under the alias + * Vector of string. String has the next format: + */ +static constexpr Property, PropertyMutability::RW> alias_for{"PROXY_ALIAS_FOR"}; + +} // namespace proxy +} // namespace ov diff --git a/src/plugins/proxy/docs/architecture.md b/src/plugins/proxy/docs/architecture.md new file mode 100644 index 00000000000..acd349cbbe1 --- /dev/null +++ b/src/plugins/proxy/docs/architecture.md @@ -0,0 +1,122 @@ +# OpenVINO Proxy Plugin architecture + +## Motivation + + - OpenVINO may have multiple hardware plugins for similar device type from different vendors (e.g. Intel and NVidia GPUs) and currently user must address them with different names ("GPU" and "NVIDIA" respectively). Using same name for such cases seems to be more user-friendly approach. + - Moreover, single physical device may have multiple plugin which support it. For example, Intel GPU plugin is OpenCL based, thus can run on other OCL-compatible devices including NVIDIA gpus. In that case we may have primary plugin ("NVIDIA") which provides best performance on target device and fallback plugin ("INTEL_GPU") which helps to improve models coverage for the cases when primary plugin has limited operation set support, and both plugins may be executed via HETERO mode. + - Implicit HETERO plugin usage may be extended to run on different device types - HETERO:xPU,CPU + + ```mermaid +flowchart TB + subgraph application [User application] + direction LR + GPU.0 + GPU.1 + GPU.2 + GPU.3 + end + + subgraph proxy_plugin [Proxy plugin] + direction LR + proxy_gpu_0[IntelGPU.0] + proxy_gpu_1[IntelGPU.1] + proxy_gpu_2[Hetero:NVIDIA.0,IntelGPU.2] + proxy_gpu_3[Hetero:NVIDIA.1,IntelGPU.3] + end + GPU.0--->proxy_gpu_0 + GPU.1--->proxy_gpu_1 + GPU.2--->proxy_gpu_2 + GPU.3--->proxy_gpu_3 + + subgraph plugins [Plugins] + direction LR + intel_gpu[Intel GPU plugin] + nvidia[NVIDIA plugin] + hetero[Hetero plugin] + end + proxy_gpu_0--->intel_gpu + proxy_gpu_1--->intel_gpu + proxy_gpu_2--->hetero + proxy_gpu_3--->hetero + hetero--->intel_gpu + hetero--->nvidia + + subgraph gpu_dev [Intel GPU Plugin devices] + direction LR + Intel_GPU.0 + Intel_GPU.1 + Intel_GPU.2 + Intel_GPU.3 + end + intel_gpu--->Intel_GPU.0 + intel_gpu--->Intel_GPU.1 + intel_gpu--->Intel_GPU.2 + intel_gpu--->Intel_GPU.3 + + subgraph nvidia_dev [NVIDIA Plugin devices] + direction LR + NVIDIA.0 + NVIDIA.1 + end + + nvidia--->NVIDIA.0 + nvidia--->NVIDIA.1 + + + subgraph hw_dev [Hardware devices] + direction LR + dev0[Intel iGPU] + dev1[Intel dGPU] + dev2[NVIDIA GPU 1] + dev3[NVIDIA GPU 2] + end + + Intel_GPU.0 ---> dev0 + Intel_GPU.1 ---> dev1 + Intel_GPU.2 ---> dev2 + Intel_GPU.3 ---> dev3 + + NVIDIA.0 ---> dev2 + NVIDIA.1 ---> dev3 +``` + +## Plugin responsibilities + + - Providing user-visible aliases which aggregates lower-level plugins ("GPU" alias for "INTEL_GPU" and "NVIDIA_GPU") + - Hide real hardware plugin names under the common device name + - Proxy plugin should provide the optimal performance with minimum overhead + - Implicit HETERO mode run for the cases when multiple plugins can be used for the same device + - Managing properties to configure target and fallback devices. + +## Requirements + + - Do not provide additional libraries and don't affect load time (proxy plugin is a part of openvino library) + ```mermaid +flowchart LR + subgraph openvino [openvino library] + core + inference + transformations[Common transformations] + lp_transformations[LP transformations] + frontend_common[Frontend common API] + + proxy_plugin[Proxy plugin] + style frontend_common fill:#7f9dc0,stroke:#333,stroke-width:4px + style transformations fill:#3d85c6,stroke:#333,stroke-width:4px + style lp_transformations fill:#0b5394,stroke:#333,stroke-width:4px + style core fill:#679f58,stroke:#333,stroke-width:4px + style inference fill:#d7a203,stroke:#333,stroke-width:4px + end +``` + - No overhead for load inference time + - Fallback to hardware plugin if device is supported only by one plugin + - Minimal overhead in case of multiple plugins for one device + - Plus one call of query network if entire model can be executed on preferable plugin + - Hetero execution in other case + - Allow to configure device + +## Behavior details + - `ov::Core` can create several instances of proxy plugins (separate instance for each high-level device) + - Proxy plugin uses `ov::device::uuid` property to match devices from different plugins + - Plugin allows to set properties of primary plugin, in case of configuration fallback plugin, user should use `ov::device::properties()` + - Plugin doesn't use the system of devices enumeration of hidden plugins and use ids for enumeration (`DEV.0`, `DEV.1`, ..., `DEV.N` and etc.) diff --git a/src/plugins/proxy/src/compiled_model.hpp b/src/plugins/proxy/src/compiled_model.hpp new file mode 100644 index 00000000000..2489624f3c8 --- /dev/null +++ b/src/plugins/proxy/src/compiled_model.hpp @@ -0,0 +1,67 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/proxy/infer_request.hpp" +#include "openvino/runtime/icompiled_model.hpp" +#include "openvino/runtime/iinfer_request.hpp" +#include "openvino/runtime/so_ptr.hpp" + +namespace ov { +namespace proxy { +class CompiledModel : public ov::ICompiledModel { +public: + CompiledModel(const ov::SoPtr& model, const std::shared_ptr& plugin) + : ov::ICompiledModel(nullptr, plugin), + m_compiled_model(model) {} + CompiledModel(const ov::SoPtr& model, + const std::shared_ptr& plugin, + const ov::RemoteContext& context) + : ov::ICompiledModel(nullptr, plugin, context), + m_compiled_model(model) {} + std::shared_ptr create_infer_request() const override { + return std::make_shared( + ov::SoPtr{m_compiled_model->create_infer_request(), m_compiled_model._so}, + shared_from_this()); + } + + void export_model(std::ostream& model) const override { + m_compiled_model->export_model(model); + } + + std::shared_ptr get_runtime_model() const override { + return m_compiled_model->get_runtime_model(); + } + + void set_property(const ov::AnyMap& properties) override { + m_compiled_model->set_property(properties); + } + + ov::Any get_property(const std::string& name) const override { + return m_compiled_model->get_property(name); + } + const std::vector>& inputs() const override { + return m_compiled_model->inputs(); + } + const std::vector>& outputs() const override { + return m_compiled_model->outputs(); + } + +protected: + /** + * @brief Method creates infer request implementation + * + * @return Sync infer request + */ + std::shared_ptr create_sync_infer_request() const override { + OPENVINO_NOT_IMPLEMENTED; + } + +private: + ov::SoPtr m_compiled_model; +}; + +} // namespace proxy +} // namespace ov diff --git a/src/plugins/proxy/src/infer_request.cpp b/src/plugins/proxy/src/infer_request.cpp new file mode 100644 index 00000000000..bcbb26a0857 --- /dev/null +++ b/src/plugins/proxy/src/infer_request.cpp @@ -0,0 +1,95 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "openvino/proxy/infer_request.hpp" + +#include "openvino/runtime/itensor.hpp" +#include "openvino/runtime/remote_context.hpp" +#include "openvino/runtime/so_ptr.hpp" +#include "remote_context.hpp" + +ov::proxy::InferRequest::InferRequest(ov::SoPtr&& request, + const std::shared_ptr& compiled_model) + : ov::IAsyncInferRequest(nullptr, nullptr, nullptr), + m_infer_request(std::move(request)), + m_compiled_model(compiled_model) {} +void ov::proxy::InferRequest::start_async() { + m_infer_request->start_async(); +} + +void ov::proxy::InferRequest::wait() { + m_infer_request->wait(); +} + +bool ov::proxy::InferRequest::wait_for(const std::chrono::milliseconds& timeout) { + return m_infer_request->wait_for(timeout); +} + +void ov::proxy::InferRequest::cancel() { + m_infer_request->cancel(); +} + +void ov::proxy::InferRequest::set_callback(std::function callback) { + m_infer_request->set_callback(callback); +} + +void ov::proxy::InferRequest::infer() { + m_infer_request->infer(); +} + +std::vector ov::proxy::InferRequest::get_profiling_info() const { + return m_infer_request->get_profiling_info(); +} + +ov::Tensor ov::proxy::InferRequest::get_tensor(const ov::Output& port) const { + auto tensor = m_infer_request->get_tensor(port); + if (tensor.is()) { + auto remote_context = std::dynamic_pointer_cast(m_compiled_model->get_context()); + OPENVINO_ASSERT(remote_context); + tensor = remote_context->wrap_tensor(tensor.as()); + } + return ov::Tensor(tensor, m_infer_request._so); +} + +void ov::proxy::InferRequest::set_tensor(const ov::Output& port, const ov::Tensor& tensor) { + m_infer_request->set_tensor(port, tensor); +} + +std::vector ov::proxy::InferRequest::get_tensors(const ov::Output& port) const { + auto tensors = m_infer_request->get_tensors(port); + for (auto&& tensor : tensors) { + if (tensor.is()) { + auto remote_context = std::dynamic_pointer_cast(m_compiled_model->get_context()); + OPENVINO_ASSERT(remote_context); + tensor = remote_context->wrap_tensor(tensor.as()); + } + tensor = ov::Tensor(tensor, m_infer_request._so); + } + return tensors; +} + +void ov::proxy::InferRequest::set_tensors(const ov::Output& port, + const std::vector& tensors) { + return m_infer_request->set_tensors(port, tensors); +} + +std::vector> ov::proxy::InferRequest::query_state() const { + return m_infer_request->query_state(); +} + +const std::shared_ptr& ov::proxy::InferRequest::get_compiled_model() const { + return m_compiled_model; +} + +const std::vector>& ov::proxy::InferRequest::get_inputs() const { + return m_infer_request->get_inputs(); +} + +const std::vector>& ov::proxy::InferRequest::get_outputs() const { + return m_infer_request->get_outputs(); +} + +const ov::SoPtr ov::proxy::InferRequest::get_hardware_request() const { + return m_infer_request; +} diff --git a/src/plugins/proxy/src/plugin.cpp b/src/plugins/proxy/src/plugin.cpp new file mode 100644 index 00000000000..0a9b142175c --- /dev/null +++ b/src/plugins/proxy/src/plugin.cpp @@ -0,0 +1,617 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "openvino/proxy/plugin.hpp" + +#include + +#include "compiled_model.hpp" +#include "cpp_interfaces/interface/ie_internal_plugin_config.hpp" +#include "openvino/core/any.hpp" +#include "openvino/core/except.hpp" +#include "openvino/proxy/properties.hpp" +#include "openvino/runtime/device_id_parser.hpp" +#include "openvino/runtime/iinfer_request.hpp" +#include "openvino/runtime/iremote_context.hpp" +#include "openvino/util/common_util.hpp" +#include "plugin.hpp" +#include "remote_context.hpp" + +namespace { + +size_t string_to_size_t(const std::string& s) { + std::stringstream sstream(s); + size_t idx; + sstream >> idx; + return idx; +} + +bool is_device_in_config(const ov::AnyMap& config) { + return config.find(ov::device::id.name()) != config.end(); +} + +ov::AnyMap remove_device_properties(ov::AnyMap& config, const std::vector& devices) { + ov::AnyMap result; + std::unordered_set devs; + for (const auto& dev : devices) + devs.insert(dev); + + for (const auto& it : config) { + auto subprop_device_name_pos = it.first.find(ov::device::properties.name() + std::string("_")); + if (subprop_device_name_pos == std::string::npos) + continue; + auto subprop_device_name = + it.first.substr(subprop_device_name_pos + std::strlen(ov::device::properties.name()) + 1); + ov::DeviceIDParser parser(subprop_device_name); + if (devs.find(subprop_device_name) != devs.end() || devs.find(parser.get_device_name()) != devs.end()) { + // It is a device property + result[subprop_device_name] = it.second; + } + } + + // Remove device properties from config + for (const auto& it : result) { + auto c_it = config.find(ov::device::properties.name() + std::string("_") + it.first); + if (c_it != config.end()) + config.erase(c_it); + } + return result; +} + +ov::AnyMap remove_proxy_properties(ov::AnyMap& config, bool rem_device_properties = false) { + const static std::vector proxy_properties = {ov::device::id, + CONFIG_KEY_INTERNAL(CONFIG_DEVICE_ID), + ov::device::priorities, + ov::proxy::alias_for, + ov::proxy::device_priorities}; + ov::AnyMap dev_properties; + for (const auto& property : proxy_properties) { + auto it = config.find(property); + if (it == config.end()) + continue; + if (ov::device::priorities == property && rem_device_properties) + dev_properties = remove_device_properties(config, it->second.as>()); + + config.erase(it); + } + + return dev_properties; +} + +// add cached properties for device configuration +ov::AnyMap construct_device_config(const std::string& dev_name, + const std::unordered_map& configs, + const ov::AnyMap& properties) { + // Initial device config should be equal to default global config + auto it = configs.find(""); + ov::AnyMap device_config = it != configs.end() ? it->second : ov::AnyMap{}; + it = configs.find(dev_name); + bool is_device = is_device_in_config(properties) && it != configs.end(); + if (is_device) { + // Adds device specific options + for (const auto& it : it->second) { + device_config[it.first] = it.second; + } + } + for (const auto& it : properties) { + device_config[it.first] = it.second; + } + remove_proxy_properties(device_config); + return device_config; +} + +} // namespace + +size_t ov::proxy::Plugin::get_device_from_config(const ov::AnyMap& config) const { + if (is_device_in_config(config)) + return config.at(ov::device::id.name()).as(); + return m_default_device; +} + +ov::SupportedOpsMap ov::proxy::Plugin::query_model(const std::shared_ptr& model, + const ov::AnyMap& properties) const { + size_t num_devices = get_hidden_devices().size(); + // Recall for HW device + auto dev_id = get_device_from_config(properties); + auto config_copy = properties; + remove_proxy_properties(config_copy); + auto res = get_core()->query_model(model, get_fallback_device(dev_id), config_copy); + // Replace hidden device name + for (auto&& it : res) { + it.second = get_device_name(); + if (num_devices > 1) + it.second += "." + std::to_string(dev_id); + } + return res; +} + +void ov::proxy::Plugin::set_property(const ov::AnyMap& properties) { + auto hw_config = properties; + // Parse default device ID and remove from config + auto it = hw_config.find(ov::device::id.name()); + if (it != hw_config.end()) { + m_default_device = it->second.as(); + hw_config.erase(it); + } + + // Replace device::id by CONFIG_DEVICE_ID + it = hw_config.find(CONFIG_KEY_INTERNAL(CONFIG_DEVICE_ID)); + if (it != hw_config.end()) { + hw_config[ov::device::id.name()] = it->second; + hw_config.erase(it); + } + // Empty config_name means means global config for all devices + std::string config_name = is_device_in_config(hw_config) ? std::to_string(get_device_from_config(hw_config)) : ""; + + // Parse alias config + it = hw_config.find(ov::proxy::alias_for.name()); + bool fill_order = hw_config.find(ov::proxy::device_priorities.name()) == hw_config.end() && m_device_order.empty(); + if (it != hw_config.end()) { + for (auto&& dev : it->second.as>()) { + m_alias_for.emplace(dev); + if (fill_order) + m_device_order.emplace_back(dev); + } + } + + // Restore device order + it = hw_config.find(ov::proxy::device_priorities.name()); + if (it != hw_config.end()) { + m_device_order.clear(); + std::vector> priority_order; + // Biggest number means minimum priority + size_t min_priority(0); + for (auto&& dev_priority : it->second.as>()) { + auto dev_prior = ov::util::split(dev_priority, ':'); + OPENVINO_ASSERT(dev_prior.size() == 2); + auto priority = string_to_size_t(dev_prior[1]); + if (priority > min_priority) + min_priority = priority; + priority_order.push_back(std::pair{dev_prior[0], priority}); + } + // Devices without priority has lower priority + min_priority++; + for (const auto& dev : m_alias_for) { + if (std::find_if(priority_order.begin(), + priority_order.end(), + [&](const std::pair& el) { + return el.first == dev; + }) == std::end(priority_order)) { + priority_order.push_back(std::pair{dev, min_priority}); + } + } + std::sort(priority_order.begin(), + priority_order.end(), + [](const std::pair& v1, const std::pair& v2) { + return v1.second < v2.second; + }); + m_device_order.reserve(priority_order.size()); + for (const auto& dev : priority_order) { + m_device_order.emplace_back(dev.first); + } + // Align sizes of device order with alias + if (m_device_order.size() < m_alias_for.size()) { + for (const auto& dev : m_alias_for) { + if (std::find(std::begin(m_device_order), std::end(m_device_order), dev) == std::end(m_device_order)) { + m_device_order.emplace_back(dev); + } + } + } + } + + { + // Cannot change config from different threads + std::lock_guard lock(m_plugin_mutex); + it = hw_config.find(ov::device::priorities.name()); + if (it != hw_config.end()) { + m_configs[config_name][ov::device::priorities.name()] = it->second; + // Main device is needed in case if we don't have alias and would like to be able change fallback order per + // device + if (m_alias_for.empty() && config_name.empty()) + m_alias_for.insert(it->second.as>()[0]); + } + } + const std::string primary_dev = get_primary_device(get_device_from_config(hw_config)); + // Add fallback priority to detect supported devices in case of HETERO fallback + auto device_priority = get_internal_property(ov::device::priorities.name(), config_name); + if (!device_priority.empty()) + hw_config[ov::device::priorities.name()] = device_priority; + auto dev_properties = remove_proxy_properties(hw_config, true); + std::string dev_prop_name; + ov::DeviceIDParser pr_parser(primary_dev); + for (const auto& it : dev_properties) { + ov::DeviceIDParser parser(it.first); + if (parser.get_device_name() == pr_parser.get_device_name()) { + // Add primary device properties to primary device + OPENVINO_ASSERT(it.second.is()); + auto dev_map = it.second.as(); + for (const auto& m_it : dev_map) { + // Plugin shouldn't contain the different property for the same key + OPENVINO_ASSERT(hw_config.find(m_it.first) == hw_config.end() || + hw_config.at(m_it.first) == m_it.second); + hw_config[m_it.first] = m_it.second; + } + dev_prop_name = it.first; + break; + } + } + { + // Cannot change config from different threads + std::lock_guard lock(m_plugin_mutex); + for (const auto& it : hw_config) { + // Skip proxy and primary device properties + if (CONFIG_KEY_INTERNAL(CONFIG_DEVICE_ID) == it.first || ov::device::id.name() == it.first || + it.first == ov::device::priorities.name() || it.first == ov::proxy::device_priorities.name() || + it.first == ov::proxy::alias_for.name() || + // Skip options from config for primaty device + hw_config.find(it.first) != hw_config.end() || (!dev_prop_name.empty() && it.first == dev_prop_name)) + continue; + // Cache proxy and fallback device options to apply for fallback devices + m_configs[config_name][it.first] = it.second; + } + } + get_core()->set_property(primary_dev, hw_config); +} + +ov::Any ov::proxy::Plugin::get_property(const std::string& name, const ov::AnyMap& arguments) const { + size_t device_id = get_device_from_config(arguments); + const std::string config_name = is_device_in_config(arguments) ? std::to_string(device_id) : ""; + if (name == ov::device::id) + return m_default_device; + + if (name == CONFIG_KEY_INTERNAL(CONFIG_DEVICE_ID)) + return std::to_string(device_id); + + if (name == ov::device::priorities) { + return get_internal_property(name, config_name).as>(); + } + if (name == ov::available_devices) { + auto hidden_devices = get_hidden_devices(); + std::vector availableDevices(hidden_devices.size()); + for (size_t i = 0; i < hidden_devices.size(); i++) { + availableDevices[i] = std::to_string(i); + } + return decltype(ov::available_devices)::value_type(availableDevices); + } + if (name == ov::supported_properties) { + auto supported_prop = + get_core()->get_property(get_primary_device(device_id), name, {}).as>(); + + // Extend primary device properties by proxy specific property + + // ov::device::id changes the default proxy device + if (std::find(supported_prop.begin(), supported_prop.end(), ov::device::id) == supported_prop.end()) + supported_prop.emplace_back(ov::device::id); + if (std::find(supported_prop.begin(), supported_prop.end(), CONFIG_KEY_INTERNAL(CONFIG_DEVICE_ID)) == + supported_prop.end()) + supported_prop.emplace_back(ov::PropertyName(CONFIG_KEY_INTERNAL(CONFIG_DEVICE_ID))); + return supported_prop; + } + + if (has_internal_property(name, config_name)) + return get_internal_property(name, config_name); + return get_core()->get_property(get_primary_device(device_id), name, {}); +} + +std::shared_ptr ov::proxy::Plugin::compile_model(const std::shared_ptr& model, + const ov::AnyMap& properties) const { + auto dev_name = get_fallback_device(get_device_from_config(properties)); + auto device_config = construct_device_config(dev_name, m_configs, properties); + std::shared_ptr plugin = shared_from_this(); + + auto compiled_model = + std::make_shared(get_core()->compile_model(model, dev_name, device_config), plugin); + return std::dynamic_pointer_cast(compiled_model); +} + +std::shared_ptr ov::proxy::Plugin::compile_model(const std::shared_ptr& model, + const ov::AnyMap& properties, + const ov::RemoteContext& context) const { + auto ctx = ov::proxy::RemoteContext::get_hardware_context(context); + auto dev_name = ctx.get_device_name(); + auto device_config = construct_device_config(dev_name, m_configs, properties); + + std::shared_ptr plugin = shared_from_this(); + auto compiled_model = + std::make_shared(get_core()->compile_model(model, ctx, device_config), + plugin, + context); + return std::dynamic_pointer_cast(compiled_model); +} + +std::shared_ptr ov::proxy::Plugin::create_context(const ov::AnyMap& remote_properties) const { + // TODO: if no device id, try to create context for each plugin + auto dev_name = get_device_name(); + auto dev_idx = get_device_from_config(remote_properties); + auto has_dev_idx = is_device_in_config(remote_properties); + auto is_new_api = get_core()->is_new_api(); + + auto device_config = remote_properties; + remove_proxy_properties(device_config); + + if (has_dev_idx) { + auto remote_context = std::make_shared( + get_core()->create_context(get_fallback_device(get_device_from_config(remote_properties)), device_config), + dev_name, + dev_idx, + has_dev_idx, + is_new_api); + return std::dynamic_pointer_cast(remote_context); + } + // Properties doesn't have device id, so try to create context for all devices + const auto hidden_devices = get_hidden_devices(); + for (size_t i = 0; i < hidden_devices.size(); i++) { + try { + auto remote_context = std::make_shared( + get_core()->create_context(get_fallback_device(get_device_from_config(remote_properties)), + device_config), + dev_name, + i, + has_dev_idx, + is_new_api); + return std::dynamic_pointer_cast(remote_context); + } catch (const ov::Exception&) { + } + } + OPENVINO_THROW("Cannot create remote context for provided properties: ", + ov::Any(remote_properties).as()); +} + +std::shared_ptr ov::proxy::Plugin::get_default_context(const ov::AnyMap& remote_properties) const { + auto dev_name = get_device_name(); + auto dev_idx = get_device_from_config(remote_properties); + auto has_dev_idx = is_device_in_config(remote_properties); + auto is_new_api = get_core()->is_new_api(); + + auto device_config = remote_properties; + remove_proxy_properties(device_config); + + auto remote_context = std::make_shared( + get_core()->get_default_context(get_fallback_device(get_device_from_config(remote_properties))), + dev_name, + dev_idx, + has_dev_idx, + is_new_api); + return std::dynamic_pointer_cast(remote_context); +} + +std::shared_ptr ov::proxy::Plugin::import_model(std::istream& model, + const ov::AnyMap& properties) const { + auto dev_name = get_fallback_device(get_device_from_config(properties)); + auto device_config = construct_device_config(dev_name, m_configs, properties); + + return std::make_shared(get_core()->import_model(model, dev_name, device_config), + shared_from_this()); +} + +std::shared_ptr ov::proxy::Plugin::import_model(std::istream& model, + const ov::RemoteContext& context, + const ov::AnyMap& properties) const { + auto ctx = ov::proxy::RemoteContext::get_hardware_context(context); + auto dev_name = ctx.get_device_name(); + auto device_config = construct_device_config(dev_name, m_configs, properties); + + return std::make_shared(get_core()->import_model(model, ctx, device_config), + shared_from_this(), + context); +} + +std::string ov::proxy::Plugin::get_primary_device(size_t idx) const { + std::vector devices; + const auto all_devices = get_hidden_devices(); + devices.reserve(all_devices.size()); + for (const auto& dev : all_devices) { + devices.emplace_back(dev.at(0)); + } + + if (devices.empty()) + // Return low level device name in case of no devices wasn't found + return m_device_order.at(0); + OPENVINO_ASSERT(devices.size() > idx); + return devices[idx]; +} + +std::string ov::proxy::Plugin::get_fallback_device(size_t idx) const { + const auto all_devices = get_hidden_devices(); + OPENVINO_ASSERT(all_devices.size() > idx); + if (all_devices[idx].size() == 1) { + return all_devices[idx].at(0); + } else { + std::string device_concatenation; + for (const auto& dev : all_devices[idx]) { + if (!device_concatenation.empty()) + device_concatenation += ","; + device_concatenation += dev; + } + return "HETERO:" + device_concatenation; + } +} + +std::vector> ov::proxy::Plugin::get_hidden_devices() const { + // Proxy plugin has 2 modes of matching devices: + // * Fallback - in this mode we report devices only for the first hidden plugin + // * Alias - Case when we group all devices under one common name + std::vector> result; + const auto core = get_core(); + OPENVINO_ASSERT(core != nullptr); + OPENVINO_ASSERT(!m_alias_for.empty()); // alias_for cannot be empty. 1 is for fallback mode, >1 in other + + // If we have 1 alias we use simple hetero mode + if (m_alias_for.size() == 1) { + auto device = *m_alias_for.begin(); + const std::vector real_devices_ids = core->get_property(device, ov::available_devices); + for (const auto& device_id : real_devices_ids) { + const std::string full_device_name = device_id.empty() ? device : device + '.' + device_id; + std::vector devices{full_device_name}; + + // Add fallback devices use device_id for individual fallback property + auto fallback = get_internal_property(ov::device::priorities.name(), device_id).as(); + if (!fallback.empty()) { + for (const auto& fallback_dev : ov::util::split(fallback, ' ')) { + devices.emplace_back(fallback_dev); + } + } + result.emplace_back(devices); + } + } else { + typedef struct DeviceId { + ov::device::UUID uuid; + std::unordered_map device_to_full_name; + bool no_uuid; + } DeviceID_t; + OPENVINO_ASSERT(m_device_order.size() == m_alias_for.size()); + + // 1. Get all available devices + // Highlevel devices list contains only unique which: + // * don't support uuid + // * uuid is unique + // 2. Use individual fallback priorities to fill each list + std::vector all_highlevel_devices; + std::set> unique_devices; + for (const auto& device : m_device_order) { + const std::vector supported_device_ids = core->get_property(device, ov::available_devices); + for (const auto& device_id : supported_device_ids) { + const std::string full_device_name = device_id.empty() ? device : device + '.' + device_id; + try { + ov::device::UUID uuid = + core->get_property(full_device_name, ov::device::uuid.name(), {}).as(); + auto it = unique_devices.find(uuid.uuid); + if (it == unique_devices.end()) { + unique_devices.insert(uuid.uuid); + DeviceID_t id; + id.no_uuid = false; + id.uuid = uuid; + id.device_to_full_name[device] = full_device_name; + all_highlevel_devices.emplace_back(id); + } else { + for (auto&& dev_id : all_highlevel_devices) { + if (dev_id.uuid.uuid == uuid.uuid) { + dev_id.device_to_full_name[device] = full_device_name; + break; + } + } + } + } catch (...) { + // Device doesn't have UUID, so it means that device is unique + DeviceID_t id; + id.no_uuid = false; + id.device_to_full_name[device] = full_device_name; + all_highlevel_devices.emplace_back(id); + } + } + } + + // Use individual fallback order to generate result list + for (size_t i = 0; i < all_highlevel_devices.size(); i++) { + std::vector real_fallback_order; + auto device = all_highlevel_devices[i]; + // In case of aliases use the proxy system of enumeration devices + const auto fallback_order = + get_internal_property(ov::device::priorities.name(), std::to_string(i)).as>(); + + bool found_primary_device = false; + bool use_hetero_mode = device.no_uuid ? true : false; + std::vector device_order; + for (const auto& fallback_dev : fallback_order) { + if (!found_primary_device) { + auto it = device.device_to_full_name.find(fallback_dev); + if (it != device.device_to_full_name.end()) { + device_order.emplace_back(it->second); + real_fallback_order.emplace_back(it->first); + found_primary_device = true; + continue; + } else { + continue; + } + } + // In case of hetero mode just add necessary devices + if (use_hetero_mode) { + device_order.emplace_back(fallback_dev); + real_fallback_order.emplace_back(fallback_dev); + continue; + } + // Try to find unique device + const std::vector supported_device_ids = + core->get_property(fallback_dev, ov::available_devices); + bool found_device = false; + bool dev_without_uuid = false; + for (const auto& device_id : supported_device_ids) { + const std::string full_device_name = fallback_dev + '.' + device_id; + try { + ov::device::UUID uuid = + core->get_property(full_device_name, ov::device::uuid.name(), {}).as(); + if (uuid.uuid == device.uuid.uuid) { + device_order.emplace_back(full_device_name); + real_fallback_order.emplace_back(fallback_dev); + found_device = true; + break; + } + } catch (...) { + dev_without_uuid = true; + } + } + // Enable hetero mode if device wasn't found + if (!found_device && dev_without_uuid) { + use_hetero_mode = true; + device_order.emplace_back(fallback_dev); + real_fallback_order.emplace_back(fallback_dev); + } + } + if (device_order.empty()) { + device_order.emplace_back(device.device_to_full_name.begin()->second); + real_fallback_order.emplace_back(device.device_to_full_name.begin()->first); + } + result.emplace_back(device_order); + std::string new_fallback; + for (const auto& dev : real_fallback_order) { + if (!new_fallback.empty()) + new_fallback += " "; + new_fallback += dev; + } + std::lock_guard lock(m_plugin_mutex); + m_configs[std::to_string(i)][ov::device::priorities.name()] = new_fallback; + } + } + return result; +} + +bool ov::proxy::Plugin::has_internal_property(const std::string& property, const std::string& config_name) const { + std::lock_guard lock(m_plugin_mutex); + auto name = config_name; + // If device specific config wasn't found or property in config wasn't found use global config + auto it = m_configs.find(name); + if (it == m_configs.end() || it->second.find(property) == it->second.end()) + name = ""; + + it = m_configs.find(name); + return (it != m_configs.end() && it->second.find(property) != it->second.end()); +} + +ov::Any ov::proxy::Plugin::get_internal_property(const std::string& property, const std::string& config_name) const { + std::lock_guard lock(m_plugin_mutex); + ov::Any result; + auto name = config_name; + // If device specific config wasn't found or property in config wasn't found use global config + auto it = m_configs.find(name); + if (it == m_configs.end() || it->second.find(property) == it->second.end()) + name = ""; + + it = m_configs.find(name); + if (it != m_configs.end() && it->second.find(property) != it->second.end()) + result = it->second.at(property); + + return result; +} + +void ov::proxy::create_plugin(::std::shared_ptr<::ov::IPlugin>& plugin) { + static const ov::Version version = {CI_BUILD_NUMBER, "openvino_proxy_plugin"}; + try { + plugin = ::std::make_shared(); + } catch (const ov::Exception&) { + throw; + } catch (const std::exception& ex) { + OPENVINO_THROW(ex.what()); + } + plugin->set_version(version); +} diff --git a/src/plugins/proxy/src/plugin.hpp b/src/plugins/proxy/src/plugin.hpp new file mode 100644 index 00000000000..34b36bd19b4 --- /dev/null +++ b/src/plugins/proxy/src/plugin.hpp @@ -0,0 +1,60 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#pragma once + +#include + +#include "openvino/runtime/iplugin.hpp" + +namespace ov { +namespace proxy { + +class Plugin : public ov::IPlugin { +public: + Plugin() = default; + ~Plugin() = default; + + void set_property(const ov::AnyMap& properties) override; + + ov::Any get_property(const std::string& name, const ov::AnyMap& arguments) const override; + + ov::SupportedOpsMap query_model(const std::shared_ptr& model, + const ov::AnyMap& properties) const override; + + std::shared_ptr compile_model(const std::shared_ptr& model, + const ov::AnyMap& properties) const override; + + std::shared_ptr compile_model(const std::shared_ptr& model, + const ov::AnyMap& properties, + const ov::RemoteContext& context) const override; + + std::shared_ptr create_context(const ov::AnyMap& remote_properties) const override; + + std::shared_ptr get_default_context(const ov::AnyMap& remote_properties) const override; + + std::shared_ptr import_model(std::istream& model, const ov::AnyMap& properties) const override; + + std::shared_ptr import_model(std::istream& model, + const ov::RemoteContext& context, + const ov::AnyMap& properties) const override; + +private: + std::vector> get_hidden_devices() const; + std::string get_fallback_device(size_t idx) const; + std::string get_primary_device(size_t idx) const; + + ov::Any get_internal_property(const std::string& property_name, const std::string& conf_name = "") const; + bool has_internal_property(const std::string& property_name, const std::string& conf_name = "") const; + size_t get_device_from_config(const ov::AnyMap& config) const; + + size_t m_default_device = 0; + std::vector m_device_order; + std::unordered_set m_alias_for; + // Update per device config in get_hidden_devices + mutable std::unordered_map m_configs; + mutable std::mutex m_plugin_mutex; +}; + +} // namespace proxy +} // namespace ov diff --git a/src/plugins/proxy/src/remote_context.cpp b/src/plugins/proxy/src/remote_context.cpp new file mode 100644 index 00000000000..b543ea97888 --- /dev/null +++ b/src/plugins/proxy/src/remote_context.cpp @@ -0,0 +1,68 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "remote_context.hpp" + +#include + +#include "openvino/proxy/plugin.hpp" +#include "openvino/runtime/iremote_context.hpp" +#include "remote_tensor.hpp" + +ov::proxy::RemoteContext::RemoteContext(ov::RemoteContext&& ctx, + const std::string& dev_name, + size_t dev_index, + bool has_index, + bool is_new_api) + : m_context(std::move(ctx)) { + m_tensor_name = dev_name + "." + std::to_string(dev_index); + // New API always has full name, in legacy API we can have device name without index + if (is_new_api || has_index) + m_name = m_tensor_name; + else + m_name = dev_name; +} + +const std::string& ov::proxy::RemoteContext::get_device_name() const { + return m_name; +} + +const ov::AnyMap& ov::proxy::RemoteContext::get_property() const { + return m_context._impl->get_property(); +} + +ov::Tensor ov::proxy::RemoteContext::wrap_tensor(const ov::RemoteTensor& tensor) { + return ov::Tensor(std::make_shared(tensor, m_tensor_name), {}); +} + +std::shared_ptr ov::proxy::RemoteContext::create_tensor(const ov::element::Type& type, + const ov::Shape& shape, + const ov::AnyMap& params) { + return std::make_shared(m_context.create_tensor(type, shape, params), m_tensor_name); +} + +std::shared_ptr ov::proxy::RemoteContext::create_host_tensor(const ov::element::Type type, + const ov::Shape& shape) { + return m_context._impl->create_host_tensor(type, shape); +} + +const ov::RemoteContext& ov::proxy::RemoteContext::get_hardware_context(const ov::RemoteContext& context) { + if (auto proxy_context = std::dynamic_pointer_cast(context._impl)) { + return proxy_context->m_context; + } + return context; +} + +const std::shared_ptr& ov::proxy::RemoteContext::get_hardware_context( + const std::shared_ptr& context) { + if (auto proxy_context = std::dynamic_pointer_cast(context)) { + return proxy_context->m_context._impl; + } + return context; +} + +const std::shared_ptr& ov::proxy::get_hardware_context( + const std::shared_ptr& context) { + return ov::proxy::RemoteContext::get_hardware_context(context); +} diff --git a/src/plugins/proxy/src/remote_context.hpp b/src/plugins/proxy/src/remote_context.hpp new file mode 100644 index 00000000000..c73c6de3985 --- /dev/null +++ b/src/plugins/proxy/src/remote_context.hpp @@ -0,0 +1,47 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#pragma once + +#include + +#include "openvino/runtime/iremote_context.hpp" +#include "openvino/runtime/iremote_tensor.hpp" +#include "openvino/runtime/remote_context.hpp" + +namespace ov { +namespace proxy { + +class RemoteContext : public ov::IRemoteContext { +public: + RemoteContext(ov::RemoteContext&& ctx, + const std::string& dev_name, + size_t dev_index, + bool has_index, + bool is_new_api); + const std::string& get_device_name() const override; + + const ov::AnyMap& get_property() const override; + + std::shared_ptr create_tensor(const ov::element::Type& type, + const ov::Shape& shape, + const ov::AnyMap& params = {}) override; + + std::shared_ptr create_host_tensor(const ov::element::Type type, const ov::Shape& shape) override; + + ov::Tensor wrap_tensor(const ov::RemoteTensor& tensor); + + static const ov::RemoteContext& get_hardware_context(const ov::RemoteContext& context); + static const std::shared_ptr& get_hardware_context( + const std::shared_ptr& context); + +private: + ov::RemoteContext m_context; + std::string m_name; + std::string m_tensor_name; + + std::string get_tensor_name() const; +}; + +} // namespace proxy +} // namespace ov diff --git a/src/plugins/proxy/src/remote_tensor.cpp b/src/plugins/proxy/src/remote_tensor.cpp new file mode 100644 index 00000000000..fe6ca6b23c9 --- /dev/null +++ b/src/plugins/proxy/src/remote_tensor.cpp @@ -0,0 +1,68 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "remote_tensor.hpp" + +#include + +#include "openvino/proxy/plugin.hpp" + +namespace { +std::shared_ptr cast_tensor(const std::shared_ptr& tensor) { + auto rem_tensor = std::dynamic_pointer_cast(tensor); + OPENVINO_ASSERT(rem_tensor); + return rem_tensor; +} +} // namespace + +ov::proxy::RemoteTensor::RemoteTensor(ov::RemoteTensor&& tensor, const std::string& dev_name) + : m_name(dev_name), + m_tensor(std::move(tensor)) {} + +ov::proxy::RemoteTensor::RemoteTensor(const ov::RemoteTensor& tensor, const std::string& dev_name) + : m_name(dev_name), + m_tensor(tensor) {} + +const ov::AnyMap& ov::proxy::RemoteTensor::get_properties() const { + return cast_tensor(m_tensor._impl)->get_properties(); +} + +const std::string& ov::proxy::RemoteTensor::get_device_name() const { + return m_name; +} + +void ov::proxy::RemoteTensor::set_shape(ov::Shape shape) { + m_tensor.set_shape(shape); +} + +const ov::element::Type& ov::proxy::RemoteTensor::get_element_type() const { + return m_tensor.get_element_type(); +} + +const ov::Shape& ov::proxy::RemoteTensor::get_shape() const { + return m_tensor.get_shape(); +} + +size_t ov::proxy::RemoteTensor::get_size() const { + return m_tensor.get_size(); +} + +size_t ov::proxy::RemoteTensor::get_byte_size() const { + return m_tensor.get_byte_size(); +} + +const ov::Strides& ov::proxy::RemoteTensor::get_strides() const { + return cast_tensor(m_tensor._impl)->get_strides(); +} + +const std::shared_ptr& ov::proxy::RemoteTensor::get_hardware_tensor( + const std::shared_ptr& tensor) { + if (auto remote_tensor = std::dynamic_pointer_cast(tensor)) + return remote_tensor->m_tensor._impl; + return tensor; +} + +const std::shared_ptr& ov::proxy::get_hardware_tensor(const std::shared_ptr& tensor) { + return ov::proxy::RemoteTensor::get_hardware_tensor(tensor); +} diff --git a/src/plugins/proxy/src/remote_tensor.hpp b/src/plugins/proxy/src/remote_tensor.hpp new file mode 100644 index 00000000000..77c2953a0d0 --- /dev/null +++ b/src/plugins/proxy/src/remote_tensor.hpp @@ -0,0 +1,42 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#pragma once + +#include + +#include "openvino/runtime/iremote_tensor.hpp" +#include "openvino/runtime/remote_tensor.hpp" + +namespace ov { +namespace proxy { + +class RemoteTensor : public ov::IRemoteTensor { +public: + RemoteTensor(ov::RemoteTensor&& ctx, const std::string& dev_name); + RemoteTensor(const ov::RemoteTensor& ctx, const std::string& dev_name); + + const AnyMap& get_properties() const override; + const std::string& get_device_name() const override; + + void set_shape(ov::Shape shape) override; + + const ov::element::Type& get_element_type() const override; + + const ov::Shape& get_shape() const override; + + size_t get_size() const override; + + size_t get_byte_size() const override; + + const ov::Strides& get_strides() const override; + + static const std::shared_ptr& get_hardware_tensor(const std::shared_ptr& tensor); + +private: + mutable std::string m_name; + ov::RemoteTensor m_tensor; +}; + +} // namespace proxy +} // namespace ov diff --git a/src/plugins/proxy/tests/CMakeLists.txt b/src/plugins/proxy/tests/CMakeLists.txt new file mode 100644 index 00000000000..1ede76945d8 --- /dev/null +++ b/src/plugins/proxy/tests/CMakeLists.txt @@ -0,0 +1,36 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +set(TARGET_NAME ov_proxy_plugin_tests) + +ov_add_test_target( + NAME ${TARGET_NAME} + ROOT ${CMAKE_CURRENT_SOURCE_DIR} + DEPENDENCIES + openvino::runtime + mock_engine + funcTestUtils + LINK_LIBRARIES + openvino::runtime::dev + gtest + gtest_main + funcTestUtils + ADD_CLANG_FORMAT + LABELS + PROXY_PLUGIN +) + +target_compile_definitions(${TARGET_NAME} PRIVATE CI_BUILD_NUMBER=\"mock_version\") + +target_include_directories(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../dev_api) + +if(ENABLE_HETERO) + add_dependencies(${TARGET_NAME} openvino_hetero_plugin) + target_compile_definitions(${TARGET_NAME} PRIVATE HETERO_ENABLED) +endif() + +if(ENABLE_OV_IR_FRONTEND) + add_dependencies(${TARGET_NAME} openvino_ir_frontend) + target_compile_definitions(${TARGET_NAME} PRIVATE IR_FRONTEND_ENABLED) +endif() diff --git a/src/plugins/proxy/tests/import_tests.cpp b/src/plugins/proxy/tests/import_tests.cpp new file mode 100644 index 00000000000..4b15e7073bc --- /dev/null +++ b/src/plugins/proxy/tests/import_tests.cpp @@ -0,0 +1,83 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "openvino/proxy/properties.hpp" +#include "proxy_tests.hpp" + +using namespace ov::proxy::tests; + +// IR frontend is needed for import +#ifdef IR_FRONTEND_ENABLED + +TEST_F(ProxyTests, import_and_infer_on_device_without_split_on_default_device) { + std::stringstream model_stream; + // Model has only add (+ 1) op and reshape + auto model = create_model_with_reshape(); + { + auto compiled_model = core.compile_model(model, "MOCK"); + compiled_model.export_model(model_stream); + } + auto compiled_model = core.import_model(model_stream, "MOCK", {}); + EXPECT_EQ(1, compiled_model.inputs().size()); + EXPECT_EQ(1, compiled_model.outputs().size()); + auto infer_request = compiled_model.create_infer_request(); + auto input_tensor = create_and_fill_tensor(model->input().get_element_type(), model->input().get_shape()); + infer_request.set_input_tensor(input_tensor); + infer_request.infer(); + auto output_tensor = infer_request.get_output_tensor(); + EXPECT_EQ(input_tensor.get_size(), output_tensor.get_size()); + EXPECT_EQ(input_tensor.get_element_type(), output_tensor.get_element_type()); + EXPECT_NE(memcmp(input_tensor.data(), output_tensor.data(), input_tensor.get_byte_size()), 0); + // Change input tensor + { + auto* data = input_tensor.data(); + for (size_t i = 0; i < input_tensor.get_size(); i++) + data[i] += 1; + } + EXPECT_EQ(memcmp(input_tensor.data(), output_tensor.data(), input_tensor.get_byte_size()), 0); +} + +TEST_F(ProxyTests, import_and_infer_on_device_without_split) { + std::stringstream model_stream; + auto model = create_model_with_subtract(); + { + auto compiled_model = core.compile_model(model, "MOCK.3"); + compiled_model.export_model(model_stream); + } + auto compiled_model = core.import_model(model_stream, "MOCK.3", {}); + EXPECT_EQ(1, compiled_model.inputs().size()); + EXPECT_EQ(1, compiled_model.outputs().size()); + auto infer_request = compiled_model.create_infer_request(); + auto input_tensor = create_and_fill_tensor(model->input().get_element_type(), model->input().get_shape()); + infer_request.set_input_tensor(input_tensor); + infer_request.infer(); + auto output_tensor = infer_request.get_output_tensor(); + EXPECT_EQ(input_tensor.get_shape(), output_tensor.get_shape()); + EXPECT_EQ(input_tensor.get_element_type(), output_tensor.get_element_type()); + EXPECT_EQ(memcmp(input_tensor.data(), output_tensor.data(), input_tensor.get_byte_size()), 0); +} + +# ifdef HETERO_ENABLED + +TEST_F(ProxyTests, import_and_infer_on_support_with_hetero_plugin) { + std::stringstream model_stream; + auto model = create_model_with_subtract(); + { + auto compiled_model = core.compile_model(model, "MOCK.1"); + compiled_model.export_model(model_stream); + } + auto compiled_model = core.import_model(model_stream, "MOCK.1", {}); + EXPECT_EQ(1, compiled_model.inputs().size()); + EXPECT_EQ(1, compiled_model.outputs().size()); + auto infer_request = compiled_model.create_infer_request(); + auto input_tensor = create_and_fill_tensor(model->input().get_element_type(), model->input().get_shape()); + infer_request.set_input_tensor(input_tensor); + infer_request.infer(); + auto output_tensor = infer_request.get_output_tensor(); + EXPECT_EQ(input_tensor.get_shape(), output_tensor.get_shape()); + EXPECT_EQ(input_tensor.get_element_type(), output_tensor.get_element_type()); + EXPECT_EQ(memcmp(input_tensor.data(), output_tensor.data(), input_tensor.get_byte_size()), 0); +} +# endif +#endif diff --git a/src/plugins/proxy/tests/load_proxy_plugin_tests.cpp b/src/plugins/proxy/tests/load_proxy_plugin_tests.cpp new file mode 100644 index 00000000000..17e6ba828ff --- /dev/null +++ b/src/plugins/proxy/tests/load_proxy_plugin_tests.cpp @@ -0,0 +1,237 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "openvino/proxy/properties.hpp" +#include "openvino/runtime/properties.hpp" +#include "proxy_tests.hpp" + +using namespace ov::proxy::tests; + +TEST_F(ProxyTests, alias_for_the_same_name) { + register_plugin_support_reshape(core, + "CBD", + {{ov::proxy::configuration::alias.name(), "CBD"}, + {ov::proxy::configuration::fallback.name(), "DEK"}, + {ov::proxy::configuration::priority.name(), 0}}); + register_plugin_support_subtract(core, "DEK", {{ov::proxy::configuration::alias.name(), "CBD"}}); + auto available_devices = core.get_available_devices(); + // 0, 1, 2 is ABC plugin + // 1, 3, 4 is BDE plugin + // ABC doesn't support subtract operation + std::unordered_map mock_reference_dev = {{"CBD.0", "CBD_ov_internal"}, + {"CBD.1", "CBD_ov_internal DEK"}, + {"CBD.2", "CBD_ov_internal"}}; + for (const auto& it : mock_reference_dev) { + EXPECT_EQ(core.get_property(it.first, ov::device::priorities), it.second); + } + for (const auto& dev : available_devices) { + auto it = mock_reference_dev.find(dev); + if (it != mock_reference_dev.end()) { + mock_reference_dev.erase(it); + } + } + // All devices should be found + EXPECT_TRUE(mock_reference_dev.empty()); +} + +TEST_F(ProxyTests, fallback_to_alias_name) { + register_plugin_support_reshape( + core, + "CBD", + {{ov::proxy::configuration::alias.name(), "CBD"}, {ov::proxy::configuration::priority.name(), 0}}); + register_plugin_support_subtract(core, + "DEK", + {{ov::proxy::configuration::alias.name(), "CBD"}, + {ov::proxy::configuration::fallback.name(), "CBD"}, + {ov::proxy::configuration::priority.name(), 1}}); + auto available_devices = core.get_available_devices(); + // 0, 1, 2 is ABC plugin + // 1, 3, 4 is BDE plugin + // ABC doesn't support subtract operation + std::unordered_map mock_reference_dev = {{"CBD.0", "CBD_ov_internal"}, + {"CBD.1", "DEK CBD_ov_internal"}, + {"CBD.2", "CBD_ov_internal"}, + {"CBD.3", "DEK"}, + {"CBD.4", "DEK"}}; + for (const auto& it : mock_reference_dev) { + EXPECT_EQ(core.get_property(it.first, ov::device::priorities), it.second); + } + for (const auto& dev : available_devices) { + auto it = mock_reference_dev.find(dev); + if (it != mock_reference_dev.end()) { + mock_reference_dev.erase(it); + } + } + // All devices should be found + EXPECT_TRUE(mock_reference_dev.empty()); +} + +TEST_F(ProxyTests, load_proxy_on_plugin_without_devices_with_the_same_name) { + auto available_devices = core.get_available_devices(); + register_plugin_without_devices( + core, + "CBD", + {{ov::proxy::configuration::alias.name(), "CBD"}, {ov::proxy::configuration::priority.name(), 0}}); + available_devices = core.get_available_devices(); + for (const auto& dev : available_devices) { + EXPECT_NE(dev, "CBD"); + } + available_devices = core.get_property("CBD", ov::available_devices); + EXPECT_TRUE(available_devices.empty()); +} + +TEST_F(ProxyTests, load_proxy_on_plugin_without_devices) { + auto available_devices = core.get_available_devices(); + register_plugin_without_devices( + core, + "Internal_CBD", + {{ov::proxy::configuration::alias.name(), "CBD"}, {ov::proxy::configuration::priority.name(), 0}}); + available_devices = core.get_available_devices(); + for (const auto& dev : available_devices) { + EXPECT_NE(dev, "CBD"); + } + available_devices = core.get_property("CBD", ov::available_devices); + EXPECT_TRUE(available_devices.empty()); +} + +TEST_F(ProxyTests, get_available_devices) { + auto available_devices = core.get_available_devices(); + // 0, 1, 2 is ABC plugin + // 1, 3, 4 is BDE plugin + // ABC doesn't support subtract operation + std::unordered_map mock_reference_dev = {{"MOCK.0", "ABC"}, + {"MOCK.1", "ABC BDE"}, + {"MOCK.2", "ABC"}, + {"MOCK.3", "BDE"}, + {"MOCK.4", "BDE"}}; + for (const auto& it : mock_reference_dev) { + EXPECT_EQ(core.get_property(it.first, ov::device::priorities), it.second); + } + for (const auto& dev : available_devices) { + EXPECT_FALSE(dev.find("ABC") != std::string::npos); + EXPECT_FALSE(dev.find("BDE") != std::string::npos); + auto it = mock_reference_dev.find(dev); + if (it != mock_reference_dev.end()) { + mock_reference_dev.erase(it); + } + } + // All devices should be found + EXPECT_TRUE(mock_reference_dev.empty()); +} + +TEST_F(ProxyTests, get_available_devices_with_low_level_plugin) { + ov::AnyMap config; + config[ov::proxy::alias_for.name()] = "BDE"; + // Change device priority + core.set_property("MOCK", config); + auto available_devices = core.get_available_devices(); + // 0, 1, 2 is ABC plugin + // 1, 3, 4 is BDE plugin + // ABC doesn't support subtract operation + { + // We don't change fallback order for hetero case + std::unordered_map mock_reference_dev = {{"MOCK.0", "ABC BDE"}, + {"MOCK.1", "ABC BDE"}, + {"MOCK.2", "ABC BDE"}}; + for (const auto& it : mock_reference_dev) { + EXPECT_EQ(core.get_property(it.first, ov::device::priorities), it.second); + } + } + std::set mock_reference_dev = {"ABC.ABC_1", "ABC.ABC_2", "ABC.ABC_3", "MOCK.0", "MOCK.1", "MOCK.2"}; + for (const auto& dev : available_devices) { + if (mock_reference_dev.find(dev) != mock_reference_dev.end()) { + mock_reference_dev.erase(dev); + } + } + // All devices should be found + EXPECT_TRUE(mock_reference_dev.empty()); +} + +TEST_F(ProxyTests, get_available_devices_with_disabled_plugin) { + ov::AnyMap config; + config[ov::device::priorities.name()] = "BDE"; + // Change device priority + core.set_property("MOCK", config); + auto available_devices = core.get_available_devices(); + std::unordered_map mock_reference_dev = {{"MOCK.0", "ABC"}, + {"MOCK.1", "BDE"}, + {"MOCK.2", "ABC"}, + {"MOCK.3", "BDE"}, + {"MOCK.4", "BDE"}}; + for (const auto& it : mock_reference_dev) { + EXPECT_EQ(core.get_property(it.first, ov::device::priorities), it.second); + } + for (const auto& dev : available_devices) { + auto it = mock_reference_dev.find(dev); + if (it != mock_reference_dev.end()) { + mock_reference_dev.erase(it); + } + } + // All devices should be found + EXPECT_TRUE(mock_reference_dev.empty()); +} + +TEST_F(ProxyTests, load_and_infer_on_device_without_split_on_default_device) { + // Model has only add (+ 1) op and reshape + auto model = create_model_with_reshape(); + auto infer_request = core.compile_model(model, "MOCK").create_infer_request(); + auto input_tensor = create_and_fill_tensor(model->input().get_element_type(), model->input().get_shape()); + infer_request.set_input_tensor(input_tensor); + infer_request.infer(); + auto output_tensor = infer_request.get_output_tensor(); + EXPECT_EQ(input_tensor.get_size(), output_tensor.get_size()); + EXPECT_EQ(input_tensor.get_element_type(), output_tensor.get_element_type()); + EXPECT_NE(memcmp(input_tensor.data(), output_tensor.data(), input_tensor.get_byte_size()), 0); + // Change input tensor + { + auto* data = input_tensor.data(); + for (size_t i = 0; i < input_tensor.get_size(); i++) + data[i] += 1; + } + EXPECT_EQ(memcmp(input_tensor.data(), output_tensor.data(), input_tensor.get_byte_size()), 0); +} + +TEST_F(ProxyTests, load_and_infer_on_device_without_split) { + auto model = create_model_with_subtract(); + auto infer_request = core.compile_model(model, "MOCK.3").create_infer_request(); + auto input_tensor = create_and_fill_tensor(model->input().get_element_type(), model->input().get_shape()); + infer_request.set_input_tensor(input_tensor); + infer_request.infer(); + auto output_tensor = infer_request.get_output_tensor(); + EXPECT_EQ(input_tensor.get_shape(), output_tensor.get_shape()); + EXPECT_EQ(input_tensor.get_element_type(), output_tensor.get_element_type()); + EXPECT_EQ(memcmp(input_tensor.data(), output_tensor.data(), input_tensor.get_byte_size()), 0); +} + +TEST_F(ProxyTests, load_on_unsupported_plugin) { + auto model = create_model_with_subtract(); + EXPECT_EQ(core.get_property("MOCK.0", ov::device::priorities), "ABC"); + EXPECT_THROW(core.compile_model(model, "MOCK.0"), ov::Exception); +} + +TEST_F(ProxyTests, load_on_supported_plugin) { + auto model = create_model_with_subtract(); + EXPECT_EQ(core.get_property("MOCK.3", ov::device::priorities), "BDE"); + EXPECT_NO_THROW(core.compile_model(model, "MOCK.3")); +} + +#ifdef HETERO_ENABLED +TEST_F(ProxyTests, load_on_shared_plugin) { + auto model = create_model_with_subtract(); + EXPECT_EQ(core.get_property("MOCK.1", ov::device::priorities), "ABC BDE"); + EXPECT_NO_THROW(core.compile_model(model, "MOCK.1")); +} + +TEST_F(ProxyTests, load_and_infer_on_support_with_hetero_plugin) { + auto model = create_model_with_subtract(); + auto infer_request = core.compile_model(model, "MOCK.1").create_infer_request(); + auto input_tensor = create_and_fill_tensor(model->input().get_element_type(), model->input().get_shape()); + infer_request.set_input_tensor(input_tensor); + infer_request.infer(); + auto output_tensor = infer_request.get_output_tensor(); + EXPECT_EQ(input_tensor.get_shape(), output_tensor.get_shape()); + EXPECT_EQ(input_tensor.get_element_type(), output_tensor.get_element_type()); + EXPECT_EQ(memcmp(input_tensor.data(), output_tensor.data(), input_tensor.get_byte_size()), 0); +} +#endif diff --git a/src/plugins/proxy/tests/properties_tests.cpp b/src/plugins/proxy/tests/properties_tests.cpp new file mode 100644 index 00000000000..b9a86bbda84 --- /dev/null +++ b/src/plugins/proxy/tests/properties_tests.cpp @@ -0,0 +1,151 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#include "proxy_tests.hpp" + +using namespace ov::proxy::tests; + +namespace { +std::string get_string_value(const ov::Any& value) { + if (value.empty()) { + return "Empty"; + } else { + return value.as(); + } +} +} // namespace + +TEST_F(ProxyTests, get_property_on_default_uninit_device) { + const std::string dev_name = "MOCK"; + EXPECT_EQ(0, core.get_property(dev_name, ov::num_streams)); + core.set_property(dev_name, ov::num_streams(2)); + EXPECT_EQ(2, core.get_property(dev_name, ov::num_streams)); +} + +TEST_F(ProxyTests, set_property_for_fallback_device) { + const std::string dev_name = "MOCK.1"; + EXPECT_EQ(0, core.get_property(dev_name, ov::num_streams)); + core.set_property(dev_name, ov::num_streams(2)); + EXPECT_EQ(2, core.get_property(dev_name, ov::num_streams)); + core.set_property(dev_name, ov::device::properties("BDE", ov::enable_profiling(true))); + EXPECT_EQ(false, core.get_property(dev_name, ov::enable_profiling)); +} + +TEST_F(ProxyTests, set_property_for_primary_device) { + const std::string dev_name = "MOCK.1"; + core.set_property(dev_name, ov::device::properties("ABC", ov::enable_profiling(true))); + EXPECT_EQ(true, core.get_property(dev_name, ov::enable_profiling)); +} + +TEST_F(ProxyTests, set_property_for_primary_device_full_name) { + const std::string dev_name = "MOCK.1"; + core.set_property(dev_name, ov::device::properties("ABC.abc_b", ov::enable_profiling(true))); + EXPECT_EQ(true, core.get_property(dev_name, ov::enable_profiling)); +} + +TEST_F(ProxyTests, get_property_on_default_device) { + const std::string dev_name = "MOCK"; + auto supported_properties = core.get_property(dev_name, ov::supported_properties); + EXPECT_EQ(10, supported_properties.size()); + size_t mutable_pr(0), immutable_pr(0); + for (auto&& property : supported_properties) { + property.is_mutable() ? mutable_pr++ : immutable_pr++; + if (property == ov::num_streams) { + EXPECT_EQ("0", get_string_value(core.get_property(dev_name, property))); + core.set_property(dev_name, ov::num_streams(2)); + EXPECT_TRUE(core.get_property(dev_name, property).is()); + EXPECT_EQ("2", get_string_value(core.get_property(dev_name, property))); + } else if (property == ov::device::uuid) { + EXPECT_EQ("000102030405060708090a0b0c0d0e0f", get_string_value(core.get_property(dev_name, property))); + } else if (property == ov::device::priorities) { + auto value = core.get_property(dev_name, property).as>(); + EXPECT_EQ(value.size(), 2); + EXPECT_EQ(value[0], "ABC"); + EXPECT_EQ(value[1], "BDE"); + } else { + EXPECT_NO_THROW(core.get_property(dev_name, property)); + } + } + EXPECT_EQ(6, immutable_pr); + EXPECT_EQ(4, mutable_pr); +} + +TEST_F(ProxyTests, get_property_on_mixed_device) { + const std::string dev_name = "MOCK.1"; + auto supported_properties = core.get_property(dev_name, ov::supported_properties); + EXPECT_EQ(10, supported_properties.size()); + size_t mutable_pr(0), immutable_pr(0); + for (auto&& property : supported_properties) { + property.is_mutable() ? mutable_pr++ : immutable_pr++; + if (property == ov::num_streams) { + EXPECT_EQ("0", get_string_value(core.get_property(dev_name, property))); + core.set_property(dev_name, ov::num_streams(2)); + EXPECT_TRUE(core.get_property(dev_name, property).is()); + EXPECT_EQ("2", get_string_value(core.get_property(dev_name, property))); + } else if (property == ov::device::uuid) { + EXPECT_EQ("00020406080a0c0e10121416181a1c1e", get_string_value(core.get_property(dev_name, property))); + } else if (property == ov::device::priorities) { + auto value = core.get_property(dev_name, property).as>(); + EXPECT_EQ(value.size(), 2); + EXPECT_EQ(value[0], "ABC"); + EXPECT_EQ(value[1], "BDE"); + } else { + core.get_property(dev_name, property); + } + } + EXPECT_EQ(6, immutable_pr); + EXPECT_EQ(4, mutable_pr); +} + +TEST_F(ProxyTests, get_property_on_specified_device) { + const std::string dev_name = "MOCK.3"; + auto supported_properties = core.get_property(dev_name, ov::supported_properties); + EXPECT_EQ(9, supported_properties.size()); + size_t mutable_pr(0), immutable_pr(0); + for (auto&& property : supported_properties) { + property.is_mutable() ? mutable_pr++ : immutable_pr++; + if (property == ov::enable_profiling) { + EXPECT_EQ("NO", get_string_value(core.get_property(dev_name, property))); + core.set_property(dev_name, ov::enable_profiling(true)); + EXPECT_TRUE(core.get_property(dev_name, property).is()); + EXPECT_EQ("YES", get_string_value(core.get_property(dev_name, property))); + } else if (property == ov::device::uuid) { + EXPECT_EQ("0004080c1014181c2024282c3034383c", get_string_value(core.get_property(dev_name, property))); + } else if (property == ov::device::priorities) { + auto value = core.get_property(dev_name, property).as>(); + EXPECT_EQ(value.size(), 1); + EXPECT_EQ(value[0], "BDE"); + } else { + EXPECT_NO_THROW(core.get_property(dev_name, property)); + } + } + EXPECT_EQ(6, immutable_pr); + EXPECT_EQ(3, mutable_pr); +} + +TEST_F(ProxyTests, get_property_for_changed_default_device) { + const std::string dev_name = "MOCK"; + core.set_property(dev_name, ov::device::id(3)); + auto supported_properties = core.get_property(dev_name, ov::supported_properties); + EXPECT_EQ(9, supported_properties.size()); + size_t mutable_pr(0), immutable_pr(0); + for (auto&& property : supported_properties) { + property.is_mutable() ? mutable_pr++ : immutable_pr++; + if (property == ov::enable_profiling) { + EXPECT_EQ("NO", get_string_value(core.get_property(dev_name, property))); + core.set_property(dev_name, ov::enable_profiling(true)); + EXPECT_TRUE(core.get_property(dev_name, property).is()); + EXPECT_EQ("YES", get_string_value(core.get_property(dev_name, property))); + } else if (property == ov::device::uuid) { + EXPECT_EQ("0004080c1014181c2024282c3034383c", get_string_value(core.get_property(dev_name, property))); + } else if (property == ov::device::priorities) { + auto value = core.get_property(dev_name, property).as>(); + EXPECT_EQ(value.size(), 1); + EXPECT_EQ(value[0], "BDE"); + } else { + EXPECT_NO_THROW(core.get_property(dev_name, property)); + } + } + EXPECT_EQ(6, immutable_pr); + EXPECT_EQ(3, mutable_pr); +} diff --git a/src/plugins/proxy/tests/proxy_tests.cpp b/src/plugins/proxy/tests/proxy_tests.cpp new file mode 100644 index 00000000000..63a8c11219a --- /dev/null +++ b/src/plugins/proxy/tests/proxy_tests.cpp @@ -0,0 +1,811 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "proxy_tests.hpp" + +#include +#include + +#include "common_test_utils/file_utils.hpp" +#include "ie_plugin_config.hpp" +#include "openvino/core/any.hpp" +#include "openvino/core/except.hpp" +#include "openvino/opsets/opset11.hpp" +#include "openvino/pass/serialize.hpp" +#include "openvino/proxy/properties.hpp" +#include "openvino/runtime/internal_properties.hpp" +#include "openvino/runtime/iplugin.hpp" +#include "openvino/runtime/iremote_context.hpp" +#include "openvino/runtime/iremote_tensor.hpp" +#include "openvino/runtime/properties.hpp" +#include "openvino/util/file_util.hpp" +#include "openvino/util/shared_object.hpp" + +namespace { + +std::string get_mock_engine_path() { + std::string mockEngineName("mock_engine"); + return ov::util::make_plugin_library_name(CommonTestUtils::getExecutableDirectory(), + mockEngineName + IE_BUILD_POSTFIX); +} + +template +std::function make_std_function(const std::shared_ptr so, const std::string& functionName) { + std::function ptr(reinterpret_cast(ov::util::get_symbol(so, functionName.c_str()))); + return ptr; +} + +bool support_model(const std::shared_ptr& model, const ov::SupportedOpsMap& supported_ops) { + for (const auto& op : model->get_ops()) { + if (supported_ops.find(op->get_friendly_name()) == supported_ops.end()) + return false; + } + return true; +} + +ov::PropertyName RO_property(const std::string& propertyName) { + return ov::PropertyName(propertyName, ov::PropertyMutability::RO); +}; + +ov::PropertyName RW_property(const std::string& propertyName) { + return ov::PropertyName(propertyName, ov::PropertyMutability::RW); +}; + +} // namespace + +void ov::proxy::tests::ProxyTests::SetUp() { + if (m_mock_plugins.empty()) { + register_plugin_support_reshape(core, + "ABC", + {{ov::proxy::configuration::alias.name(), "MOCK"}, + {ov::proxy::configuration::fallback.name(), "BDE"}, + {ov::proxy::configuration::priority.name(), 0}}); + register_plugin_support_subtract(core, "BDE", {{ov::proxy::configuration::alias.name(), "MOCK"}}); + } +} + +ov::Tensor ov::proxy::tests::ProxyTests::create_and_fill_tensor(const ov::element::Type& type, const ov::Shape& shape) { + switch (type) { + case ov::element::Type_t::i64: + return create_tensor::value_type>(type, shape); + default: + break; + } + OPENVINO_THROW("Cannot generate tensor. Unsupported element type."); +} + +std::shared_ptr ov::proxy::tests::ProxyTests::create_model_with_subtract() { + auto param = std::make_shared(ov::element::i64, ov::Shape{1, 3, 2, 2}); + param->set_friendly_name("input"); + auto const_value = ov::opset11::Constant::create(ov::element::i64, ov::Shape{1, 1, 1, 1}, {1}); + const_value->set_friendly_name("const_val"); + auto add = std::make_shared(param, const_value); + add->set_friendly_name("add"); + auto subtract = std::make_shared(add, const_value); + subtract->set_friendly_name("sub"); + auto result = std::make_shared(subtract); + result->set_friendly_name("res"); + return std::make_shared(ov::ResultVector{result}, ov::ParameterVector{param}); +} + +std::shared_ptr ov::proxy::tests::ProxyTests::create_model_with_subtract_reshape() { + auto param = std::make_shared(ov::element::i64, ov::Shape{1, 3, 2, 2}); + param->set_friendly_name("input"); + auto const_value = ov::opset11::Constant::create(ov::element::i64, ov::Shape{1, 1, 1, 1}, {1}); + const_value->set_friendly_name("const_val"); + auto add = std::make_shared(param, const_value); + add->set_friendly_name("add"); + auto subtract = std::make_shared(add, const_value); + subtract->set_friendly_name("sub"); + auto reshape_val = ov::opset11::Constant::create(ov::element::i64, ov::Shape{1}, {-1}); + reshape_val->set_friendly_name("reshape_val"); + auto reshape = std::make_shared(subtract, reshape_val, true); + reshape->set_friendly_name("reshape"); + auto result = std::make_shared(reshape); + result->set_friendly_name("res"); + return std::make_shared(ov::ResultVector{result}, ov::ParameterVector{param}); +} + +std::shared_ptr ov::proxy::tests::ProxyTests::create_model_with_subtract_reshape_relu() { + auto param = std::make_shared(ov::element::i64, ov::Shape{1, 3, 2, 2}); + param->set_friendly_name("input"); + auto const_value = ov::opset11::Constant::create(ov::element::i64, ov::Shape{1, 1, 1, 1}, {1}); + const_value->set_friendly_name("const_val"); + auto add = std::make_shared(param, const_value); + add->set_friendly_name("add"); + auto subtract = std::make_shared(add, const_value); + subtract->set_friendly_name("sub"); + auto reshape_val = ov::opset11::Constant::create(ov::element::i64, ov::Shape{1}, {-1}); + reshape_val->set_friendly_name("reshape_val"); + auto reshape = std::make_shared(subtract, reshape_val, true); + reshape->set_friendly_name("reshape"); + auto relu = std::make_shared(reshape); + relu->set_friendly_name("relu"); + auto result = std::make_shared(relu); + result->set_friendly_name("res"); + return std::make_shared(ov::ResultVector{result}, ov::ParameterVector{param}); +} + +std::shared_ptr ov::proxy::tests::ProxyTests::create_model_with_reshape() { + auto param = std::make_shared(ov::element::i64, ov::Shape{1, 3, 2, 2}); + param->set_friendly_name("input"); + auto const_value = ov::opset11::Constant::create(ov::element::i64, ov::Shape{1, 1, 1, 1}, {1}); + const_value->set_friendly_name("const_val"); + auto add = std::make_shared(param, const_value); + add->set_friendly_name("add"); + auto reshape_val = ov::opset11::Constant::create(ov::element::i64, ov::Shape{1}, {-1}); + reshape_val->set_friendly_name("reshape_val"); + auto reshape = std::make_shared(add, reshape_val, true); + reshape->set_friendly_name("reshape"); + auto result = std::make_shared(reshape); + result->set_friendly_name("res"); + return std::make_shared(ov::ResultVector{result}, ov::ParameterVector{param}); +} + +// Mock plugins + +class MockCompiledModel : public ov::ICompiledModel { +public: + MockCompiledModel(const std::shared_ptr& model, + const std::shared_ptr& plugin, + const ov::AnyMap& config) + : ov::ICompiledModel(model, plugin), + m_config(config), + m_model(model), + m_has_context(false) {} + + MockCompiledModel(const std::shared_ptr& model, + const std::shared_ptr& plugin, + const ov::AnyMap& config, + const ov::RemoteContext& context) + : ov::ICompiledModel(model, plugin), + m_config(config), + m_model(model), + m_has_context(true), + m_context(context) {} + + // Methods from a base class ov::ICompiledModel + void export_model(std::ostream& model) const override { + ov::pass::StreamSerialize(model, std::function()) + .run_on_model(std::const_pointer_cast(m_model)); + } + + std::shared_ptr get_runtime_model() const override { + OPENVINO_NOT_IMPLEMENTED; + } + + void set_property(const ov::AnyMap& properties) override { + OPENVINO_NOT_IMPLEMENTED; + } + + ov::Any get_property(const std::string& name) const override { + OPENVINO_NOT_IMPLEMENTED; + } + + std::shared_ptr create_sync_infer_request() const override; + + const std::shared_ptr& get_model() const { + return m_model; + } + + ov::RemoteContext get_context() const { + return m_context; + } + + bool has_context() const { + return m_has_context; + } + +private: + ov::AnyMap m_config; + std::shared_ptr m_model; + bool m_has_context; + ov::RemoteContext m_context; +}; + +class MockInferRequest : public ov::ISyncInferRequest { +public: + MockInferRequest(const std::shared_ptr& compiled_model) + : ov::ISyncInferRequest(compiled_model) { + OPENVINO_ASSERT(compiled_model); + m_model = compiled_model->get_model(); + // Allocate input/output tensors + for (const auto& input : get_inputs()) { + allocate_tensor(input, [this, input, compiled_model](ov::Tensor& tensor) { + // Can add a check to avoid double work in case of shared tensors + allocate_tensor_impl(tensor, + input.get_element_type(), + input.get_partial_shape().is_dynamic() ? ov::Shape{0} : input.get_shape(), + compiled_model->has_context(), + compiled_model->get_context()); + }); + } + for (const auto& output : get_outputs()) { + allocate_tensor(output, [this, output, compiled_model](ov::Tensor& tensor) { + // Can add a check to avoid double work in case of shared tensors + allocate_tensor_impl(tensor, + output.get_element_type(), + output.get_partial_shape().is_dynamic() ? ov::Shape{0} : output.get_shape(), + compiled_model->has_context(), + compiled_model->get_context()); + }); + } + } + ~MockInferRequest() = default; + + void infer() override { + ov::TensorVector input_tensors; + for (const auto& input : get_inputs()) { + input_tensors.emplace_back(get_tensor(input)); + } + ov::TensorVector output_tensors; + for (const auto& output : get_outputs()) { + output_tensors.emplace_back(get_tensor(output)); + } + m_model->evaluate(output_tensors, input_tensors); + } + std::vector> query_state() const override { + OPENVINO_NOT_IMPLEMENTED; + } + std::vector get_profiling_info() const override { + OPENVINO_NOT_IMPLEMENTED; + } + +private: + void allocate_tensor_impl(ov::Tensor& tensor, + const ov::element::Type& element_type, + const ov::Shape& shape, + bool has_context, + ov::RemoteContext context) { + if (!tensor || tensor.get_element_type() != element_type) { + if (has_context) { + tensor = context.create_tensor(element_type, shape, {}); + } else { + tensor = ov::Tensor(element_type, shape); + } + } else { + tensor.set_shape(shape); + } + } + std::shared_ptr m_model; +}; + +std::shared_ptr MockCompiledModel::create_sync_infer_request() const { + return std::make_shared(std::dynamic_pointer_cast(shared_from_this())); +} + +class MockRemoteTensor : public ov::IRemoteTensor { + ov::AnyMap m_properties; + std::string m_dev_name; + +public: + MockRemoteTensor(const std::string& name, const ov::AnyMap& props) : m_properties(props), m_dev_name(name) {} + const ov::AnyMap& get_properties() const override { + return m_properties; + } + const std::string& get_device_name() const override { + return m_dev_name; + } + void set_shape(ov::Shape shape) override { + OPENVINO_NOT_IMPLEMENTED; + } + + const ov::element::Type& get_element_type() const override { + OPENVINO_NOT_IMPLEMENTED; + } + + const ov::Shape& get_shape() const override { + OPENVINO_NOT_IMPLEMENTED; + } + + const ov::Strides& get_strides() const override { + OPENVINO_NOT_IMPLEMENTED; + } +}; + +class MockRemoteContext : public ov::IRemoteContext { + ov::AnyMap m_property = {{"IS_DEFAULT", true}}; + std::string m_dev_name; + +public: + MockRemoteContext(const std::string& dev_name) : m_dev_name(dev_name) {} + const std::string& get_device_name() const override { + return m_dev_name; + } + + const ov::AnyMap& get_property() const override { + return m_property; + } + + std::shared_ptr create_tensor(const ov::element::Type& type, + const ov::Shape& shape, + const ov::AnyMap& params = {}) override { + auto remote_tensor = std::make_shared(m_dev_name, m_property); + return remote_tensor; + } +}; + +class MockCustomRemoteContext : public ov::IRemoteContext { + ov::AnyMap m_property = {{"IS_DEFAULT", false}}; + std::string m_dev_name; + +public: + MockCustomRemoteContext(const std::string& dev_name) : m_dev_name(dev_name) {} + const std::string& get_device_name() const override { + return m_dev_name; + } + + const ov::AnyMap& get_property() const override { + return m_property; + } + + std::shared_ptr create_tensor(const ov::element::Type& type, + const ov::Shape& shape, + const ov::AnyMap& params = {}) override { + auto remote_tensor = std::make_shared(m_dev_name, m_property); + return remote_tensor; + } +}; + +class MockPluginBase : public ov::IPlugin { +public: + virtual const ov::Version& get_const_version() = 0; + + std::shared_ptr compile_model(const std::shared_ptr& model, + const ov::AnyMap& properties) const override { + OPENVINO_ASSERT(model); + if (!support_model(model, query_model(model, properties))) + OPENVINO_THROW("Unsupported model"); + + return std::make_shared(model, shared_from_this(), properties); + } + + std::shared_ptr compile_model(const std::string& model_path, + const ov::AnyMap& properties) const override { + OPENVINO_NOT_IMPLEMENTED; + } + + std::shared_ptr compile_model(const std::shared_ptr& model, + const ov::AnyMap& properties, + const ov::RemoteContext& context) const override { + if (!support_model(model, query_model(model, properties))) + OPENVINO_THROW("Unsupported model"); + + return std::make_shared(model, shared_from_this(), properties, context); + } + + void set_property(const ov::AnyMap& properties) override { + OPENVINO_NOT_IMPLEMENTED; + } + + ov::Any get_property(const std::string& name, const ov::AnyMap& arguments) const override { + OPENVINO_NOT_IMPLEMENTED; + } + + std::shared_ptr create_context(const ov::AnyMap& remote_properties) const override { + if (remote_properties.find("CUSTOM_CTX") == remote_properties.end()) + return std::make_shared(get_device_name()); + return std::make_shared(get_device_name()); + } + + std::shared_ptr get_default_context(const ov::AnyMap& remote_properties) const override { + return std::make_shared(get_device_name()); + } + + std::shared_ptr import_model(std::istream& model, const ov::AnyMap& properties) const override { + std::string xmlString, xmlInOutString; + ov::Tensor weights; + + ov::pass::StreamSerialize::DataHeader hdr = {}; + model.read(reinterpret_cast(&hdr), sizeof hdr); + + // read CNNNetwork input/output precisions + model.seekg(hdr.custom_data_offset); + xmlInOutString.resize(hdr.custom_data_size); + model.read(const_cast(xmlInOutString.c_str()), hdr.custom_data_size); + + // read blob content + model.seekg(hdr.consts_offset); + if (hdr.consts_size) { + weights = ov::Tensor(ov::element::i8, ov::Shape{hdr.consts_size}); + char* data = static_cast(weights.data()); + model.read(data, hdr.consts_size); + } + + // read XML content + model.seekg(hdr.model_offset); + xmlString.resize(hdr.model_size); + model.read(const_cast(xmlString.c_str()), hdr.model_size); + + ov::Core core; + auto ov_model = core.read_model(xmlString, weights); + return compile_model(ov_model, properties); + } + + std::shared_ptr import_model(std::istream& model, + const ov::RemoteContext& context, + const ov::AnyMap& properties) const override { + std::string xmlString, xmlInOutString; + ov::Tensor weights; + + ov::pass::StreamSerialize::DataHeader hdr = {}; + model.read(reinterpret_cast(&hdr), sizeof hdr); + + // read CNNNetwork input/output precisions + model.seekg(hdr.custom_data_offset); + xmlInOutString.resize(hdr.custom_data_size); + model.read(const_cast(xmlInOutString.c_str()), hdr.custom_data_size); + + // read blob content + model.seekg(hdr.consts_offset); + if (hdr.consts_size) { + weights = ov::Tensor(ov::element::i8, ov::Shape{hdr.consts_size}); + char* data = static_cast(weights.data()); + model.read(data, hdr.consts_size); + } + + // read XML content + model.seekg(hdr.model_offset); + xmlString.resize(hdr.model_size); + model.read(const_cast(xmlString.c_str()), hdr.model_size); + + ov::Core core; + auto ov_model = core.read_model(xmlString, weights); + return compile_model(ov_model, properties, context); + } + + ov::SupportedOpsMap query_model(const std::shared_ptr& model, + const ov::AnyMap& properties) const override { + OPENVINO_NOT_IMPLEMENTED; + } +}; + +void ov::proxy::tests::ProxyTests::reg_plugin(ov::Core& core, + std::shared_ptr& plugin, + const std::string& device_name, + const ov::AnyMap& properties) { + std::string libraryPath = get_mock_engine_path(); + if (!m_so) + m_so = ov::util::load_shared_object(libraryPath.c_str()); + if (auto mock_plugin = std::dynamic_pointer_cast(plugin)) + mock_plugin->set_version(mock_plugin->get_const_version()); + plugin->set_device_name(device_name); + std::function injectProxyEngine = make_std_function(m_so, "InjectPlugin"); + + injectProxyEngine(plugin.get()); + core.register_plugin(ov::util::make_plugin_library_name(CommonTestUtils::getExecutableDirectory(), + std::string("mock_engine") + IE_BUILD_POSTFIX), + device_name, + properties); + m_mock_plugins.emplace_back(plugin); +} + +// test +void ov::proxy::tests::ProxyTests::register_plugin_support_reshape(ov::Core& core, + const std::string& device_name, + const ov::AnyMap& properties) { + class MockPluginReshape : public MockPluginBase { + public: + const ov::Version& get_const_version() override { + static const ov::Version version = {CI_BUILD_NUMBER, "openvino_mock_reshape_plugin"}; + return version; + } + ov::SupportedOpsMap query_model(const std::shared_ptr& model, + const ov::AnyMap& properties) const override { + OPENVINO_ASSERT(model); + + std::unordered_set supported_ops = {"Parameter", "Result", "Add", "Constant", "Reshape"}; + + ov::SupportedOpsMap res; + for (const auto& op : model->get_ordered_ops()) { + if (supported_ops.find(op->get_type_info().name) == supported_ops.end()) + continue; + res.emplace(op->get_friendly_name(), get_device_name()); + } + return res; + } + + void set_property(const ov::AnyMap& properties) override { + for (const auto& it : properties) { + if (it.first == ov::num_streams.name()) + num_streams = it.second.as(); + else if (it.first == ov::enable_profiling.name()) + m_profiling = it.second.as(); + else if (it.first == ov::device::id.name()) + continue; + else + OPENVINO_THROW(get_device_name(), " set config: " + it.first); + } + } + + ov::Any get_property(const std::string& name, const ov::AnyMap& arguments) const override { + const static std::vector device_ids = {get_device_name() + "_1", + get_device_name() + "_2", + get_device_name() + "_3"}; + const static std::vector roProperties{ + RO_property(ov::supported_properties.name()), + RO_property(ov::available_devices.name()), + RO_property(ov::loaded_from_cache.name()), + RO_property(ov::device::uuid.name()), + RO_property(ov::caching_properties.name()), + RO_property(METRIC_KEY(IMPORT_EXPORT_SUPPORT)), + }; + // the whole config is RW before network is loaded. + const static std::vector rwProperties{ + RW_property(ov::num_streams.name()), + RW_property(ov::enable_profiling.name()), + }; + + std::string device_id; + if (arguments.find(ov::device::id.name()) != arguments.end()) { + device_id = arguments.find(ov::device::id.name())->second.as(); + } + if (name == ov::supported_properties) { + std::vector supportedProperties; + supportedProperties.reserve(roProperties.size() + rwProperties.size()); + supportedProperties.insert(supportedProperties.end(), roProperties.begin(), roProperties.end()); + supportedProperties.insert(supportedProperties.end(), rwProperties.begin(), rwProperties.end()); + + return decltype(ov::supported_properties)::value_type(supportedProperties); + } else if (name == ov::device::uuid) { + ov::device::UUID uuid; + for (size_t i = 0; i < uuid.MAX_UUID_SIZE; i++) { + if (device_id == device_ids[0]) + uuid.uuid[i] = static_cast(i); + else if (device_id == device_ids[1]) + uuid.uuid[i] = static_cast(i * 2); + else if (device_id == device_ids[2]) + uuid.uuid[i] = static_cast(i * 3); + } + return decltype(ov::device::uuid)::value_type{uuid}; + } else if (name == ov::available_devices) { + return decltype(ov::available_devices)::value_type(device_ids); + } else if (name == ov::device::capabilities) { + std::vector capabilities; + capabilities.push_back(ov::device::capability::EXPORT_IMPORT); + return decltype(ov::device::capabilities)::value_type(capabilities); + } else if (name == "SUPPORTED_CONFIG_KEYS") { // TODO: Remove this key + std::vector configs; + for (const auto& property : rwProperties) { + configs.emplace_back(property); + } + return configs; + } else if (METRIC_KEY(IMPORT_EXPORT_SUPPORT) == name) { + return true; + } else if (ov::caching_properties == name) { + std::vector caching_properties = {ov::device::uuid}; + return decltype(ov::caching_properties)::value_type(caching_properties); + } else if (name == "SUPPORTED_METRICS") { // TODO: Remove this key + std::vector configs; + for (const auto& property : roProperties) { + configs.emplace_back(property); + } + return configs; + } else if (name == ov::loaded_from_cache.name()) { + return m_loaded_from_cache; + } else if (name == ov::enable_profiling.name()) { + return decltype(ov::enable_profiling)::value_type{m_profiling}; + } else if (name == ov::streams::num.name()) { + return decltype(ov::streams::num)::value_type{num_streams}; + } + OPENVINO_THROW("Unsupported property: ", name); + } + + private: + int32_t num_streams{0}; + bool m_profiling = false; + bool m_loaded_from_cache{false}; + }; + + auto plugin = std::make_shared(); + + std::shared_ptr base_plugin = plugin; + + reg_plugin(core, base_plugin, device_name, properties); +} + +void ov::proxy::tests::ProxyTests::register_plugin_support_subtract(ov::Core& core, + const std::string& device_name, + const ov::AnyMap& properties) { + class MockPluginSubtract : public MockPluginBase { + public: + const ov::Version& get_const_version() override { + static const ov::Version version = {CI_BUILD_NUMBER, "openvino_mock_subtract_plugin"}; + return version; + } + ov::SupportedOpsMap query_model(const std::shared_ptr& model, + const ov::AnyMap& properties) const override { + OPENVINO_ASSERT(model); + + std::unordered_set supported_ops = {"Parameter", "Result", "Add", "Constant", "Subtract"}; + + ov::SupportedOpsMap res; + for (const auto& op : model->get_ordered_ops()) { + if (supported_ops.find(op->get_type_info().name) == supported_ops.end()) + continue; + res[op->get_friendly_name()] = get_device_name(); + } + return res; + } + + void set_property(const ov::AnyMap& properties) override { + for (const auto& it : properties) { + if (it.first == ov::enable_profiling.name()) + m_profiling = it.second.as(); + else if (it.first == ov::device::id.name()) + continue; + else + OPENVINO_THROW(get_device_name(), " set config: " + it.first); + } + } + + ov::Any get_property(const std::string& name, const ov::AnyMap& arguments) const override { + const static std::vector device_ids = {get_device_name() + "_1", + get_device_name() + "_2", + get_device_name() + "_3"}; + const static std::vector roProperties{ + RO_property(ov::supported_properties.name()), + RO_property(ov::available_devices.name()), + RO_property(ov::loaded_from_cache.name()), + RO_property(ov::device::uuid.name()), + RO_property(ov::caching_properties.name()), + RO_property(METRIC_KEY(IMPORT_EXPORT_SUPPORT)), + }; + // the whole config is RW before network is loaded. + const static std::vector rwProperties{ + RW_property(ov::enable_profiling.name()), + }; + std::string device_id; + if (arguments.find(ov::device::id.name()) != arguments.end()) { + device_id = arguments.find(ov::device::id.name())->second.as(); + } + if (name == ov::supported_properties) { + std::vector supportedProperties; + supportedProperties.reserve(roProperties.size() + rwProperties.size()); + supportedProperties.insert(supportedProperties.end(), roProperties.begin(), roProperties.end()); + supportedProperties.insert(supportedProperties.end(), rwProperties.begin(), rwProperties.end()); + + return decltype(ov::supported_properties)::value_type(supportedProperties); + } else if (name == ov::device::uuid) { + ov::device::UUID uuid; + for (size_t i = 0; i < uuid.MAX_UUID_SIZE; i++) { + if (device_id == device_ids[0]) + uuid.uuid[i] = static_cast(i * 2); + else if (device_id == device_ids[1]) + uuid.uuid[i] = static_cast(i * 4); + else if (device_id == device_ids[2]) + uuid.uuid[i] = static_cast(i * 5); + } + return decltype(ov::device::uuid)::value_type{uuid}; + } else if (name == ov::available_devices) { + return decltype(ov::available_devices)::value_type(device_ids); + } else if (name == ov::device::capabilities) { + std::vector capabilities; + capabilities.push_back(ov::device::capability::EXPORT_IMPORT); + return decltype(ov::device::capabilities)::value_type(capabilities); + } else if (name == ov::loaded_from_cache.name()) { + return m_loaded_from_cache; + } else if (name == ov::enable_profiling.name()) { + return decltype(ov::enable_profiling)::value_type{m_profiling}; + } else if (name == "SUPPORTED_CONFIG_KEYS") { // TODO: Remove this key + std::vector configs; + for (const auto& property : rwProperties) { + configs.emplace_back(property); + } + return configs; + } else if (METRIC_KEY(IMPORT_EXPORT_SUPPORT) == name) { + return true; + } else if (ov::caching_properties == name) { + std::vector caching_properties = {ov::device::uuid}; + return decltype(ov::caching_properties)::value_type(caching_properties); + } else if (name == "SUPPORTED_METRICS") { // TODO: Remove this key + std::vector configs; + for (const auto& property : roProperties) { + configs.emplace_back(property); + } + return configs; + } + OPENVINO_THROW("Unsupported property: ", name); + } + + private: + bool m_profiling{false}; + bool m_loaded_from_cache{false}; + }; + auto plugin = std::make_shared(); + + std::shared_ptr base_plugin = plugin; + + reg_plugin(core, base_plugin, device_name, properties); +} + +void ov::proxy::tests::ProxyTests::register_plugin_without_devices(ov::Core& core, + const std::string& device_name, + const ov::AnyMap& properties) { + class MockPluginNoDevices : public MockPluginBase { + public: + const ov::Version& get_const_version() override { + static const ov::Version version = {CI_BUILD_NUMBER, "openvino_mock_no_devices_plugin"}; + return version; + } + ov::SupportedOpsMap query_model(const std::shared_ptr& model, + const ov::AnyMap& properties) const override { + OPENVINO_ASSERT(model); + + OPENVINO_NOT_IMPLEMENTED; + } + + void set_property(const ov::AnyMap& properties) override { + for (const auto& it : properties) { + if (it.first == ov::enable_profiling.name()) + m_profiling = it.second.as(); + else if (it.first == ov::device::id.name()) + continue; + else + OPENVINO_THROW(get_device_name(), " set config: " + it.first); + } + } + + ov::Any get_property(const std::string& name, const ov::AnyMap& arguments) const override { + const static std::vector device_ids = {}; + const static std::vector roProperties{ + RO_property(ov::supported_properties.name()), + RO_property(ov::available_devices.name()), + RO_property(ov::loaded_from_cache.name()), + RO_property(ov::caching_properties.name()), + RO_property(METRIC_KEY(IMPORT_EXPORT_SUPPORT)), + }; + // the whole config is RW before network is loaded. + const static std::vector rwProperties{ + RW_property(ov::enable_profiling.name()), + }; + std::string device_id; + if (arguments.find(ov::device::id.name()) != arguments.end()) { + device_id = arguments.find(ov::device::id.name())->second.as(); + } + if (name == ov::supported_properties) { + std::vector supportedProperties; + supportedProperties.reserve(roProperties.size() + rwProperties.size()); + supportedProperties.insert(supportedProperties.end(), roProperties.begin(), roProperties.end()); + supportedProperties.insert(supportedProperties.end(), rwProperties.begin(), rwProperties.end()); + + return decltype(ov::supported_properties)::value_type(supportedProperties); + } else if (name == ov::available_devices) { + return decltype(ov::available_devices)::value_type(device_ids); + } else if (name == ov::device::capabilities) { + std::vector capabilities; + capabilities.push_back(ov::device::capability::EXPORT_IMPORT); + return decltype(ov::device::capabilities)::value_type(capabilities); + } else if (name == ov::loaded_from_cache.name()) { + return m_loaded_from_cache; + } else if (name == ov::enable_profiling.name()) { + return decltype(ov::enable_profiling)::value_type{m_profiling}; + } else if (name == "SUPPORTED_CONFIG_KEYS") { // TODO: Remove this key + std::vector configs; + for (const auto& property : rwProperties) { + configs.emplace_back(property); + } + return configs; + } else if (METRIC_KEY(IMPORT_EXPORT_SUPPORT) == name) { + return true; + } else if (name == "SUPPORTED_METRICS") { // TODO: Remove this key + std::vector configs; + for (const auto& property : roProperties) { + configs.emplace_back(property); + } + return configs; + } + OPENVINO_THROW("Unsupported property: ", name); + } + + private: + bool m_profiling{false}; + bool m_loaded_from_cache{false}; + }; + auto plugin = std::make_shared(); + + std::shared_ptr base_plugin = plugin; + + reg_plugin(core, base_plugin, device_name, properties); +} diff --git a/src/plugins/proxy/tests/proxy_tests.hpp b/src/plugins/proxy/tests/proxy_tests.hpp new file mode 100644 index 00000000000..9bebfe0a36f --- /dev/null +++ b/src/plugins/proxy/tests/proxy_tests.hpp @@ -0,0 +1,99 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#pragma once + +#include + +#include + +#include "openvino/runtime/core.hpp" + +namespace ov { +namespace proxy { +namespace tests { + +class PluginRemoteTensor : public ov::RemoteTensor { +public: + /** + * @brief Checks that type defined runtime parameters are presented in remote object + * @param tensor a tensor to check + */ + static void type_check(const Tensor& tensor) { + RemoteTensor::type_check(tensor, {{"IS_DEFAULT", {}}}); + } + + bool is_default() { + return get_params().at("IS_DEFAULT").as(); + } +}; + +class PluginRemoteContext : public ov::RemoteContext { +public: + // Needed to make create_tensor overloads from base class visible for user + using RemoteContext::create_host_tensor; + using RemoteContext::create_tensor; + /** + * @brief Checks that type defined runtime parameters are presented in remote object + * @param remote_context A remote context to check + */ + static void type_check(const RemoteContext& remote_context) { + RemoteContext::type_check(remote_context, {{"IS_DEFAULT", {}}}); + } + + bool is_default() { + return get_params().at("IS_DEFAULT").as(); + } +}; + +// +// +// +// +// // The same by default +// // NO by default +// +// +// +// +// +// +class ProxyTests : public ::testing::Test { +public: + ov::Core core; + + void SetUp() override; + + std::shared_ptr create_model_with_subtract(); + std::shared_ptr create_model_with_subtract_reshape(); + std::shared_ptr create_model_with_subtract_reshape_relu(); + std::shared_ptr create_model_with_reshape(); + ov::Tensor create_and_fill_tensor(const ov::element::Type& type, const ov::Shape& shape); + +protected: + void register_plugin_without_devices(ov::Core& core, const std::string& device_name, const ov::AnyMap& properties); + void register_plugin_support_reshape(ov::Core& core, const std::string& device_name, const ov::AnyMap& properties); + void register_plugin_support_subtract(ov::Core& core, const std::string& device_name, const ov::AnyMap& properties); + +private: + template + ov::Tensor create_tensor(const ov::element::Type& type, const ov::Shape& shape) { + ov::Tensor tensor(type, shape); + T* data = tensor.data(); + for (size_t i = 0; i < tensor.get_size(); i++) { + data[i] = static_cast(i); + } + return tensor; + } + std::vector> m_mock_plugins; + std::shared_ptr m_so; + + void reg_plugin(ov::Core& core, + std::shared_ptr& plugin, + const std::string& device_name, + const ov::AnyMap& properties); +}; + +} // namespace tests +} // namespace proxy +} // namespace ov diff --git a/src/plugins/proxy/tests/query_model_tests.cpp b/src/plugins/proxy/tests/query_model_tests.cpp new file mode 100644 index 00000000000..ccc07f2c10d --- /dev/null +++ b/src/plugins/proxy/tests/query_model_tests.cpp @@ -0,0 +1,57 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#include "proxy_tests.hpp" + +using namespace ov::proxy::tests; + +// AVAILABLE_DEVICES {"MOCK.0", "MOCK.1", "MOCK.2", "MOCK.3", "MOCK.4"}; +// 1 is shared device +TEST_F(ProxyTests, query_model_on_abc) { + const std::string dev_name = "MOCK.0"; + const auto model = create_model_with_subtract_reshape(); + auto supported_ops = core.query_model(model, dev_name); + std::unordered_set names; + for (const auto& op : model->get_ops()) { + names.insert(op->get_friendly_name()); + } + for (const auto& op : supported_ops) { + EXPECT_EQ(op.second, dev_name); + names.erase(op.first); + } + EXPECT_EQ(1, names.size()); + EXPECT_EQ("sub", *names.begin()); +} + +TEST_F(ProxyTests, query_model_on_bde) { + const std::string dev_name = "MOCK.4"; + const auto model = create_model_with_subtract_reshape(); + auto supported_ops = core.query_model(model, dev_name); + std::unordered_set names; + for (const auto& op : model->get_ops()) { + names.insert(op->get_friendly_name()); + } + for (const auto& op : supported_ops) { + EXPECT_EQ(op.second, dev_name); + names.erase(op.first); + } + EXPECT_EQ(1, names.size()); + EXPECT_EQ("reshape", *names.begin()); +} + +#ifdef HETERO_ENABLED +TEST_F(ProxyTests, query_model_on_mixed) { + const std::string dev_name = "MOCK.1"; + const auto model = create_model_with_subtract_reshape(); + auto supported_ops = core.query_model(model, dev_name); + std::unordered_set names; + for (const auto& op : model->get_ops()) { + names.insert(op->get_friendly_name()); + } + for (const auto& op : supported_ops) { + EXPECT_EQ(op.second, dev_name); + names.erase(op.first); + } + EXPECT_EQ(0, names.size()); +} +#endif diff --git a/src/plugins/proxy/tests/remote_context_tests.cpp b/src/plugins/proxy/tests/remote_context_tests.cpp new file mode 100644 index 00000000000..2ff861d9dd4 --- /dev/null +++ b/src/plugins/proxy/tests/remote_context_tests.cpp @@ -0,0 +1,118 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#include "proxy_tests.hpp" + +using namespace ov::proxy::tests; + +TEST_F(ProxyTests, get_default_context_from_default_dev) { + const std::string dev_name = "MOCK"; + auto context = core.get_default_context(dev_name); + EXPECT_EQ("MOCK.0", context.get_device_name()); + ASSERT_TRUE(context.is()); + auto rem_context = context.as(); + EXPECT_TRUE(rem_context.is_default()); + auto model = create_model_with_reshape(); + + auto compiled_model = core.compile_model(model, context); + auto comp_context = compiled_model.get_context(); + EXPECT_EQ("MOCK.0", comp_context.get_device_name()); + ASSERT_TRUE(comp_context.is()); + auto rem_comp_context = comp_context.as(); + EXPECT_TRUE(rem_comp_context.is_default()); +} + +TEST_F(ProxyTests, get_default_context_from_main_dev) { + const std::string dev_name = "MOCK.0"; + auto context = core.get_default_context(dev_name); + EXPECT_EQ("MOCK.0", context.get_device_name()); + ASSERT_TRUE(context.is()); + auto rem_context = context.as(); + EXPECT_TRUE(rem_context.is_default()); + auto model = create_model_with_reshape(); + + auto compiled_model = core.compile_model(model, context); + auto comp_context = compiled_model.get_context(); + EXPECT_EQ("MOCK.0", comp_context.get_device_name()); + ASSERT_TRUE(comp_context.is()); + auto rem_comp_context = comp_context.as(); + EXPECT_TRUE(rem_comp_context.is_default()); +} + +TEST_F(ProxyTests, get_default_context_from_splited_dev) { + const std::string dev_name = "MOCK.1"; + EXPECT_ANY_THROW(auto context = core.get_default_context(dev_name)); +} + +TEST_F(ProxyTests, get_default_context_from_second_dev) { + const std::string dev_name = "MOCK.3"; + auto context = core.get_default_context(dev_name); + EXPECT_EQ("MOCK.3", context.get_device_name()); + ASSERT_TRUE(context.is()); + auto rem_context = context.as(); + EXPECT_TRUE(rem_context.is_default()); + auto model = create_model_with_subtract(); + + auto compiled_model = core.compile_model(model, context); + auto comp_context = compiled_model.get_context(); + EXPECT_EQ("MOCK.3", comp_context.get_device_name()); + ASSERT_TRUE(comp_context.is()); + auto rem_comp_context = comp_context.as(); + EXPECT_TRUE(rem_comp_context.is_default()); +} + +TEST_F(ProxyTests, create_custom_context_from_default_dev) { + const std::string dev_name = "MOCK"; + auto context = core.create_context(dev_name, {{"CUSTOM_CTX", true}}); + EXPECT_EQ("MOCK.0", context.get_device_name()); + ASSERT_TRUE(context.is()); + auto rem_context = context.as(); + EXPECT_FALSE(rem_context.is_default()); + auto model = create_model_with_reshape(); + + auto compiled_model = core.compile_model(model, context); + auto comp_context = compiled_model.get_context(); + EXPECT_EQ("MOCK.0", comp_context.get_device_name()); + ASSERT_TRUE(comp_context.is()); + auto rem_comp_context = comp_context.as(); + EXPECT_FALSE(rem_comp_context.is_default()); +} + +TEST_F(ProxyTests, create_custom_context_from_main_dev) { + const std::string dev_name = "MOCK.0"; + auto context = core.create_context(dev_name, {{"CUSTOM_CTX", true}}); + EXPECT_EQ("MOCK.0", context.get_device_name()); + ASSERT_TRUE(context.is()); + auto rem_context = context.as(); + EXPECT_FALSE(rem_context.is_default()); + auto model = create_model_with_reshape(); + + auto compiled_model = core.compile_model(model, context); + auto comp_context = compiled_model.get_context(); + EXPECT_EQ("MOCK.0", comp_context.get_device_name()); + ASSERT_TRUE(comp_context.is()); + auto rem_comp_context = comp_context.as(); + EXPECT_FALSE(rem_comp_context.is_default()); +} + +TEST_F(ProxyTests, create_custom_context_from_splited_dev) { + const std::string dev_name = "MOCK.1"; + EXPECT_ANY_THROW(auto context = core.create_context(dev_name, {{"CUSTOM_CTX", true}})); +} + +TEST_F(ProxyTests, create_custom_context_from_second_dev) { + const std::string dev_name = "MOCK.3"; + auto context = core.create_context(dev_name, {{"CUSTOM_CTX", true}}); + EXPECT_EQ("MOCK.3", context.get_device_name()); + ASSERT_TRUE(context.is()); + auto rem_context = context.as(); + EXPECT_FALSE(rem_context.is_default()); + auto model = create_model_with_subtract(); + + auto compiled_model = core.compile_model(model, context); + auto comp_context = compiled_model.get_context(); + EXPECT_EQ("MOCK.3", comp_context.get_device_name()); + ASSERT_TRUE(comp_context.is()); + auto rem_comp_context = comp_context.as(); + EXPECT_FALSE(rem_comp_context.is_default()); +} diff --git a/src/plugins/proxy/tests/remote_tensor_tests.cpp b/src/plugins/proxy/tests/remote_tensor_tests.cpp new file mode 100644 index 00000000000..355668c7a9b --- /dev/null +++ b/src/plugins/proxy/tests/remote_tensor_tests.cpp @@ -0,0 +1,174 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#include "proxy_tests.hpp" + +using namespace ov::proxy::tests; + +TEST_F(ProxyTests, default_tensor_from_default_dev) { + const std::string dev_name = "MOCK"; + auto context = core.get_default_context(dev_name); + auto tensor = context.create_tensor(ov::element::f32, {}); + EXPECT_EQ("MOCK.0", tensor.get_device_name()); + ASSERT_TRUE(tensor.is()); + auto rem_tensor = tensor.as(); + EXPECT_TRUE(rem_tensor.is_default()); + auto model = create_model_with_reshape(); + + auto compiled_model = core.compile_model(model, context); + auto comp_context = compiled_model.get_context(); + auto comp_tensor = comp_context.create_tensor(ov::element::f32, {}); + EXPECT_EQ("MOCK.0", comp_tensor.get_device_name()); + ASSERT_TRUE(comp_tensor.is()); + auto comp_rem_tensor = comp_tensor.as(); + EXPECT_TRUE(comp_rem_tensor.is_default()); + + auto infer_request = compiled_model.create_infer_request(); + auto in_tensor = infer_request.get_input_tensor(); + ASSERT_TRUE(in_tensor.is()); + auto in_rem_tensor = in_tensor.as(); + EXPECT_EQ("MOCK.0", in_rem_tensor.get_device_name()); + ASSERT_TRUE(in_rem_tensor.is()); + auto casted_in_rem_tensor = in_rem_tensor.as(); + EXPECT_TRUE(casted_in_rem_tensor.is_default()); +} + +TEST_F(ProxyTests, default_tensor_from_main_dev) { + const std::string dev_name = "MOCK.0"; + auto context = core.get_default_context(dev_name); + auto tensor = context.create_tensor(ov::element::f32, {}); + EXPECT_EQ("MOCK.0", tensor.get_device_name()); + ASSERT_TRUE(tensor.is()); + auto rem_tensor = tensor.as(); + EXPECT_TRUE(rem_tensor.is_default()); + auto model = create_model_with_reshape(); + + auto compiled_model = core.compile_model(model, context); + auto comp_context = compiled_model.get_context(); + auto comp_tensor = comp_context.create_tensor(ov::element::f32, {}); + EXPECT_EQ("MOCK.0", comp_tensor.get_device_name()); + ASSERT_TRUE(comp_tensor.is()); + auto comp_rem_tensor = comp_tensor.as(); + EXPECT_TRUE(comp_rem_tensor.is_default()); + + auto infer_request = compiled_model.create_infer_request(); + auto in_tensor = infer_request.get_input_tensor(); + ASSERT_TRUE(in_tensor.is()); + auto in_rem_tensor = in_tensor.as(); + EXPECT_EQ("MOCK.0", in_rem_tensor.get_device_name()); + ASSERT_TRUE(in_rem_tensor.is()); + auto casted_in_rem_tensor = in_rem_tensor.as(); + EXPECT_TRUE(casted_in_rem_tensor.is_default()); +} + +TEST_F(ProxyTests, default_tensor_from_second_dev) { + const std::string dev_name = "MOCK.3"; + auto context = core.get_default_context(dev_name); + auto tensor = context.create_tensor(ov::element::f32, {}); + EXPECT_EQ("MOCK.3", tensor.get_device_name()); + ASSERT_TRUE(tensor.is()); + auto rem_tensor = tensor.as(); + EXPECT_TRUE(rem_tensor.is_default()); + auto model = create_model_with_subtract(); + + auto compiled_model = core.compile_model(model, context); + auto comp_context = compiled_model.get_context(); + auto comp_tensor = comp_context.create_tensor(ov::element::f32, {}); + EXPECT_EQ("MOCK.3", comp_tensor.get_device_name()); + ASSERT_TRUE(comp_tensor.is()); + auto comp_rem_tensor = comp_tensor.as(); + EXPECT_TRUE(comp_rem_tensor.is_default()); + + auto infer_request = compiled_model.create_infer_request(); + auto in_tensor = infer_request.get_input_tensor(); + ASSERT_TRUE(in_tensor.is()); + auto in_rem_tensor = in_tensor.as(); + EXPECT_EQ("MOCK.3", in_rem_tensor.get_device_name()); + ASSERT_TRUE(in_rem_tensor.is()); + auto casted_in_rem_tensor = in_rem_tensor.as(); + EXPECT_TRUE(casted_in_rem_tensor.is_default()); +} + +TEST_F(ProxyTests, custom_tensor_from_default_dev) { + const std::string dev_name = "MOCK"; + auto context = core.create_context(dev_name, {{"CUSTOM_CTX", true}}); + auto tensor = context.create_tensor(ov::element::f32, {}); + EXPECT_EQ("MOCK.0", tensor.get_device_name()); + ASSERT_TRUE(tensor.is()); + auto rem_tensor = tensor.as(); + EXPECT_FALSE(rem_tensor.is_default()); + auto model = create_model_with_reshape(); + + auto compiled_model = core.compile_model(model, context); + auto comp_context = compiled_model.get_context(); + auto comp_tensor = comp_context.create_tensor(ov::element::f32, {}); + EXPECT_EQ("MOCK.0", comp_tensor.get_device_name()); + ASSERT_TRUE(comp_tensor.is()); + auto comp_rem_tensor = comp_tensor.as(); + EXPECT_FALSE(comp_rem_tensor.is_default()); + + auto infer_request = compiled_model.create_infer_request(); + auto in_tensor = infer_request.get_input_tensor(); + ASSERT_TRUE(in_tensor.is()); + auto in_rem_tensor = in_tensor.as(); + EXPECT_EQ("MOCK.0", in_rem_tensor.get_device_name()); + ASSERT_TRUE(in_rem_tensor.is()); + auto casted_in_rem_tensor = in_rem_tensor.as(); + EXPECT_FALSE(casted_in_rem_tensor.is_default()); +} + +TEST_F(ProxyTests, custom_tensor_from_main_dev) { + const std::string dev_name = "MOCK.0"; + auto context = core.create_context(dev_name, {{"CUSTOM_CTX", true}}); + auto tensor = context.create_tensor(ov::element::f32, {}); + EXPECT_EQ("MOCK.0", tensor.get_device_name()); + ASSERT_TRUE(tensor.is()); + auto rem_tensor = tensor.as(); + EXPECT_FALSE(rem_tensor.is_default()); + auto model = create_model_with_reshape(); + + auto compiled_model = core.compile_model(model, context); + auto comp_context = compiled_model.get_context(); + auto comp_tensor = comp_context.create_tensor(ov::element::f32, {}); + EXPECT_EQ("MOCK.0", comp_tensor.get_device_name()); + ASSERT_TRUE(comp_tensor.is()); + auto comp_rem_tensor = comp_tensor.as(); + EXPECT_FALSE(comp_rem_tensor.is_default()); + + auto infer_request = compiled_model.create_infer_request(); + auto in_tensor = infer_request.get_input_tensor(); + ASSERT_TRUE(in_tensor.is()); + auto in_rem_tensor = in_tensor.as(); + EXPECT_EQ("MOCK.0", in_rem_tensor.get_device_name()); + ASSERT_TRUE(in_rem_tensor.is()); + auto casted_in_rem_tensor = in_rem_tensor.as(); + EXPECT_FALSE(casted_in_rem_tensor.is_default()); +} + +TEST_F(ProxyTests, custom_tensor_from_second_dev) { + const std::string dev_name = "MOCK.3"; + auto context = core.create_context(dev_name, {{"CUSTOM_CTX", true}}); + auto tensor = context.create_tensor(ov::element::f32, {}); + EXPECT_EQ("MOCK.3", tensor.get_device_name()); + ASSERT_TRUE(tensor.is()); + auto rem_tensor = tensor.as(); + EXPECT_FALSE(rem_tensor.is_default()); + auto model = create_model_with_subtract(); + + auto compiled_model = core.compile_model(model, context); + auto comp_context = compiled_model.get_context(); + auto comp_tensor = comp_context.create_tensor(ov::element::f32, {}); + EXPECT_EQ("MOCK.3", comp_tensor.get_device_name()); + ASSERT_TRUE(comp_tensor.is()); + auto comp_rem_tensor = comp_tensor.as(); + EXPECT_FALSE(comp_rem_tensor.is_default()); + + auto infer_request = compiled_model.create_infer_request(); + auto in_tensor = infer_request.get_input_tensor(); + ASSERT_TRUE(in_tensor.is()); + auto in_rem_tensor = in_tensor.as(); + EXPECT_EQ("MOCK.3", in_rem_tensor.get_device_name()); + ASSERT_TRUE(in_rem_tensor.is()); + auto casted_in_rem_tensor = in_rem_tensor.as(); + EXPECT_FALSE(casted_in_rem_tensor.is_default()); +} diff --git a/src/tests/ie_test_utils/unit_test_utils/mocks/cpp_interfaces/interface/mock_icore.hpp b/src/tests/ie_test_utils/unit_test_utils/mocks/cpp_interfaces/interface/mock_icore.hpp index d53490ee91d..b24b2a8e2de 100644 --- a/src/tests/ie_test_utils/unit_test_utils/mocks/cpp_interfaces/interface/mock_icore.hpp +++ b/src/tests/ie_test_utils/unit_test_utils/mocks/cpp_interfaces/interface/mock_icore.hpp @@ -93,6 +93,7 @@ public: MOCK_CONST_METHOD1(get_default_context, ov::RemoteContext(const std::string&)); MOCK_CONST_METHOD3(import_model, ov::SoPtr(std::istream&, const ov::RemoteContext&, const ov::AnyMap&)); + MOCK_METHOD2(set_property, void(const std::string& device_name, const ov::AnyMap& properties)); ~MockICore() = default; };