From 8463eee2f6e46fd0bb7cc5113989afd44b090de5 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Wed, 3 Apr 2024 01:31:11 -0700 Subject: [PATCH] Upstream Intel NPU Plugin (#23537) ### Details: - Add ENABLE_INTEL_NPU option - Add ENABLE_NPU_DEBUG_CAPS option - Add npu plugin codes to src/plugins/intel_npu/ - Add src/inference/include/openvino/runtime/intel_npu/properties.hpp - Add thirdparty level-zero and level-zero-ext to src/plugins/intel_npu/thirdparty - Export openvino::openvino_npu_al and openvino::openvino_npu_logger_utils ### Tickets: - E-115936 --------- Co-authored-by: Ilya Lavrenov --- .../workflows/job_cpu_functional_tests.yml | 2 + .../linux_conditional_compilation.yml | 1 + .../windows_conditional_compilation.yml | 10 +- .gitmodules | 6 + cmake/features.cmake | 9 + cmake/packaging/debian.cmake | 2 +- cmake/packaging/rpm.cmake | 4 +- .../openvino/runtime/intel_npu/properties.hpp | 57 + src/plugins/intel_npu/CMakeLists.txt | 29 + src/plugins/intel_npu/README.md | 218 +++ src/plugins/intel_npu/cmake/features.cmake | 18 + .../intel_npu/docs/img/high_level_design.png | 3 + .../intel_npu/docs/img/stateful_models.png | 3 + src/plugins/intel_npu/src/CMakeLists.txt | 22 + src/plugins/intel_npu/src/al/CMakeLists.txt | 39 + .../src/al/include/device_helpers.hpp | 16 + .../al/include/intel_npu/al/config/common.hpp | 249 ++++ .../include/intel_npu/al/config/compiler.hpp | 363 +++++ .../al/include/intel_npu/al/config/config.hpp | 444 +++++++ .../include/intel_npu/al/config/runtime.hpp | 185 +++ .../include/intel_npu/al/icompiled_model.hpp | 37 + .../src/al/include/intel_npu/al/icompiler.hpp | 141 ++ .../src/al/include/intel_npu/al/itt.hpp | 20 + .../src/al/include/intel_npu/al/prefix.hpp | 28 + .../src/al/include/intel_npu/al/profiling.hpp | 40 + src/plugins/intel_npu/src/al/include/npu.hpp | 74 ++ .../src/al/include/npu_private_properties.hpp | 347 +++++ .../src/al/include/sync_infer_request.hpp | 186 +++ .../src/al/include/variable_state.hpp | 34 + .../intel_npu/src/al/src/config/common.cpp | 55 + .../intel_npu/src/al/src/config/compiler.cpp | 121 ++ .../intel_npu/src/al/src/config/config.cpp | 241 ++++ .../intel_npu/src/al/src/config/runtime.cpp | 130 ++ .../intel_npu/src/al/src/device_helpers.cpp | 33 + src/plugins/intel_npu/src/al/src/npu.cpp | 51 + .../src/al/src/sync_infer_request.cpp | 186 +++ .../intel_npu/src/backend/CMakeLists.txt | 52 + .../src/backend/include/zero_backend.hpp | 31 + .../src/backend/include/zero_device.hpp | 53 + .../src/backend/include/zero_executor.hpp | 76 ++ .../backend/include/zero_infer_request.hpp | 51 + .../src/backend/include/zero_init.hpp | 57 + .../src/backend/include/zero_memory.hpp | 121 ++ .../src/backend/include/zero_pipeline.hpp | 38 + .../src/backend/include/zero_profiling.hpp | 112 ++ .../src/backend/include/zero_types.hpp | 129 ++ .../src/backend/include/zero_utils.hpp | 219 +++ .../src/backend/include/zero_wrappers.hpp | 155 +++ .../src/backend/src/zero_backend.cpp | 46 + .../intel_npu/src/backend/src/zero_device.cpp | 125 ++ .../src/backend/src/zero_executor.cpp | 105 ++ .../src/backend/src/zero_infer_request.cpp | 264 ++++ .../intel_npu/src/backend/src/zero_init.cpp | 152 +++ .../intel_npu/src/backend/src/zero_memory.cpp | 108 ++ .../src/backend/src/zero_pipeline.cpp | 288 ++++ .../src/backend/src/zero_profiling.cpp | 249 ++++ .../src/backend/src/zero_wrappers.cpp | 150 +++ .../intel_npu/src/compiler/CMakeLists.txt | 37 + .../include/graph_transformations.hpp | 22 + .../compiler/include/iexternal_compiler.hpp | 44 + .../include/npu_driver_compiler_adapter.hpp | 47 + .../include/zero_compiler_in_driver.hpp | 220 +++ .../compiler/src/graph_transformations.cpp | 56 + .../src/npu_driver_compiler_adapter.cpp | 233 ++++ .../intel_npu/src/compiler/src/precomp.hpp | 35 + .../compiler/src/zero_compiler_in_driver.cpp | 1181 +++++++++++++++++ .../intel_npu/src/plugin/CMakeLists.txt | 66 + .../plugin/include/async_infer_request.hpp | 34 + .../src/plugin/include/compiled_model.hpp | 67 + .../src/plugin/include/npu_backends.hpp | 42 + .../src/plugin/include/npu_compiler.hpp | 18 + .../src/plugin/include/npu_metrics.hpp | 73 + .../intel_npu/src/plugin/include/plugin.hpp | 74 ++ .../src/plugin/src/async_infer_request.cpp | 27 + .../src/plugin/src/compiled_model.cpp | 330 +++++ .../intel_npu/src/plugin/src/plugin.cpp | 649 +++++++++ .../src/plugin/src/vpux_backends.cpp | 201 +++ .../src/plugin/src/vpux_compiler.cpp | 92 ++ .../intel_npu/src/plugin/src/vpux_metrics.cpp | 150 +++ .../intel_npu/src/utils/CMakeLists.txt | 7 + .../include/intel_npu/utils/logger/logger.hpp | 110 ++ .../intel_npu/utils/zero/zero_result.hpp | 17 + .../intel_npu/src/utils/src/CMakeLists.txt | 6 + .../src/utils/src/logger/CMakeLists.txt | 35 + .../intel_npu/src/utils/src/logger/logger.cpp | 151 +++ .../src/utils/src/zero/CMakeLists.txt | 23 + .../src/utils/src/zero/zero_result.cpp | 328 +++++ .../intel_npu/thirdparty/CMakeLists.txt | 36 + src/plugins/intel_npu/thirdparty/level-zero | 1 + .../intel_npu/thirdparty/level-zero-ext | 1 + 90 files changed, 10391 insertions(+), 7 deletions(-) create mode 100644 src/inference/include/openvino/runtime/intel_npu/properties.hpp create mode 100644 src/plugins/intel_npu/CMakeLists.txt create mode 100644 src/plugins/intel_npu/README.md create mode 100644 src/plugins/intel_npu/cmake/features.cmake create mode 100644 src/plugins/intel_npu/docs/img/high_level_design.png create mode 100644 src/plugins/intel_npu/docs/img/stateful_models.png create mode 100644 src/plugins/intel_npu/src/CMakeLists.txt create mode 100644 src/plugins/intel_npu/src/al/CMakeLists.txt create mode 100644 src/plugins/intel_npu/src/al/include/device_helpers.hpp create mode 100644 src/plugins/intel_npu/src/al/include/intel_npu/al/config/common.hpp create mode 100644 src/plugins/intel_npu/src/al/include/intel_npu/al/config/compiler.hpp create mode 100644 src/plugins/intel_npu/src/al/include/intel_npu/al/config/config.hpp create mode 100644 src/plugins/intel_npu/src/al/include/intel_npu/al/config/runtime.hpp create mode 100644 src/plugins/intel_npu/src/al/include/intel_npu/al/icompiled_model.hpp create mode 100644 src/plugins/intel_npu/src/al/include/intel_npu/al/icompiler.hpp create mode 100644 src/plugins/intel_npu/src/al/include/intel_npu/al/itt.hpp create mode 100644 src/plugins/intel_npu/src/al/include/intel_npu/al/prefix.hpp create mode 100644 src/plugins/intel_npu/src/al/include/intel_npu/al/profiling.hpp create mode 100644 src/plugins/intel_npu/src/al/include/npu.hpp create mode 100644 src/plugins/intel_npu/src/al/include/npu_private_properties.hpp create mode 100644 src/plugins/intel_npu/src/al/include/sync_infer_request.hpp create mode 100644 src/plugins/intel_npu/src/al/include/variable_state.hpp create mode 100644 src/plugins/intel_npu/src/al/src/config/common.cpp create mode 100644 src/plugins/intel_npu/src/al/src/config/compiler.cpp create mode 100644 src/plugins/intel_npu/src/al/src/config/config.cpp create mode 100644 src/plugins/intel_npu/src/al/src/config/runtime.cpp create mode 100644 src/plugins/intel_npu/src/al/src/device_helpers.cpp create mode 100644 src/plugins/intel_npu/src/al/src/npu.cpp create mode 100644 src/plugins/intel_npu/src/al/src/sync_infer_request.cpp create mode 100644 src/plugins/intel_npu/src/backend/CMakeLists.txt create mode 100644 src/plugins/intel_npu/src/backend/include/zero_backend.hpp create mode 100644 src/plugins/intel_npu/src/backend/include/zero_device.hpp create mode 100644 src/plugins/intel_npu/src/backend/include/zero_executor.hpp create mode 100644 src/plugins/intel_npu/src/backend/include/zero_infer_request.hpp create mode 100644 src/plugins/intel_npu/src/backend/include/zero_init.hpp create mode 100644 src/plugins/intel_npu/src/backend/include/zero_memory.hpp create mode 100644 src/plugins/intel_npu/src/backend/include/zero_pipeline.hpp create mode 100644 src/plugins/intel_npu/src/backend/include/zero_profiling.hpp create mode 100644 src/plugins/intel_npu/src/backend/include/zero_types.hpp create mode 100644 src/plugins/intel_npu/src/backend/include/zero_utils.hpp create mode 100644 src/plugins/intel_npu/src/backend/include/zero_wrappers.hpp create mode 100644 src/plugins/intel_npu/src/backend/src/zero_backend.cpp create mode 100644 src/plugins/intel_npu/src/backend/src/zero_device.cpp create mode 100644 src/plugins/intel_npu/src/backend/src/zero_executor.cpp create mode 100644 src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp create mode 100644 src/plugins/intel_npu/src/backend/src/zero_init.cpp create mode 100644 src/plugins/intel_npu/src/backend/src/zero_memory.cpp create mode 100644 src/plugins/intel_npu/src/backend/src/zero_pipeline.cpp create mode 100644 src/plugins/intel_npu/src/backend/src/zero_profiling.cpp create mode 100644 src/plugins/intel_npu/src/backend/src/zero_wrappers.cpp create mode 100644 src/plugins/intel_npu/src/compiler/CMakeLists.txt create mode 100644 src/plugins/intel_npu/src/compiler/include/graph_transformations.hpp create mode 100644 src/plugins/intel_npu/src/compiler/include/iexternal_compiler.hpp create mode 100644 src/plugins/intel_npu/src/compiler/include/npu_driver_compiler_adapter.hpp create mode 100644 src/plugins/intel_npu/src/compiler/include/zero_compiler_in_driver.hpp create mode 100644 src/plugins/intel_npu/src/compiler/src/graph_transformations.cpp create mode 100644 src/plugins/intel_npu/src/compiler/src/npu_driver_compiler_adapter.cpp create mode 100644 src/plugins/intel_npu/src/compiler/src/precomp.hpp create mode 100644 src/plugins/intel_npu/src/compiler/src/zero_compiler_in_driver.cpp create mode 100644 src/plugins/intel_npu/src/plugin/CMakeLists.txt create mode 100644 src/plugins/intel_npu/src/plugin/include/async_infer_request.hpp create mode 100644 src/plugins/intel_npu/src/plugin/include/compiled_model.hpp create mode 100644 src/plugins/intel_npu/src/plugin/include/npu_backends.hpp create mode 100644 src/plugins/intel_npu/src/plugin/include/npu_compiler.hpp create mode 100644 src/plugins/intel_npu/src/plugin/include/npu_metrics.hpp create mode 100644 src/plugins/intel_npu/src/plugin/include/plugin.hpp create mode 100644 src/plugins/intel_npu/src/plugin/src/async_infer_request.cpp create mode 100644 src/plugins/intel_npu/src/plugin/src/compiled_model.cpp create mode 100644 src/plugins/intel_npu/src/plugin/src/plugin.cpp create mode 100644 src/plugins/intel_npu/src/plugin/src/vpux_backends.cpp create mode 100644 src/plugins/intel_npu/src/plugin/src/vpux_compiler.cpp create mode 100644 src/plugins/intel_npu/src/plugin/src/vpux_metrics.cpp create mode 100644 src/plugins/intel_npu/src/utils/CMakeLists.txt create mode 100644 src/plugins/intel_npu/src/utils/include/intel_npu/utils/logger/logger.hpp create mode 100644 src/plugins/intel_npu/src/utils/include/intel_npu/utils/zero/zero_result.hpp create mode 100644 src/plugins/intel_npu/src/utils/src/CMakeLists.txt create mode 100644 src/plugins/intel_npu/src/utils/src/logger/CMakeLists.txt create mode 100644 src/plugins/intel_npu/src/utils/src/logger/logger.cpp create mode 100644 src/plugins/intel_npu/src/utils/src/zero/CMakeLists.txt create mode 100644 src/plugins/intel_npu/src/utils/src/zero/zero_result.cpp create mode 100644 src/plugins/intel_npu/thirdparty/CMakeLists.txt create mode 160000 src/plugins/intel_npu/thirdparty/level-zero create mode 160000 src/plugins/intel_npu/thirdparty/level-zero-ext diff --git a/.github/workflows/job_cpu_functional_tests.yml b/.github/workflows/job_cpu_functional_tests.yml index 5936a028be5..c3c3b57e1b5 100644 --- a/.github/workflows/job_cpu_functional_tests.yml +++ b/.github/workflows/job_cpu_functional_tests.yml @@ -100,6 +100,8 @@ jobs: if [[ -f "${INSTALL_DIR}/setupvars.sh" ]]; then source ${INSTALL_DIR}/setupvars.sh fi + # Needed as ze_loader.so is under INSTALL_TEST_DIR + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${INSTALL_TEST_DIR} python3 ${PARALLEL_TEST_SCRIPT} -e ${INSTALL_TEST_DIR}/ov_cpu_func_tests -c ${PARALLEL_TEST_CACHE} -w ${INSTALL_TEST_DIR} -s suite -rf 0 -- --gtest_print_time=1 --gtest_filter=*smoke* timeout-minutes: 25 diff --git a/.github/workflows/linux_conditional_compilation.yml b/.github/workflows/linux_conditional_compilation.yml index f42f04f6d80..b664d8771e4 100644 --- a/.github/workflows/linux_conditional_compilation.yml +++ b/.github/workflows/linux_conditional_compilation.yml @@ -212,6 +212,7 @@ jobs: tar -czvf ${BUILD_DIR}/openvino_tests.tar.gz \ tests/ov_cpu_func_tests \ tests/libopenvino_template_extension.so \ + tests/libze_loader.so* \ tests/functional_test_utils/layer_tests_summary/* popd diff --git a/.github/workflows/windows_conditional_compilation.yml b/.github/workflows/windows_conditional_compilation.yml index 3b893eb65b2..722d58ee9e7 100644 --- a/.github/workflows/windows_conditional_compilation.yml +++ b/.github/workflows/windows_conditional_compilation.yml @@ -198,13 +198,13 @@ jobs: - name: Ctest - OpenVINO unit tests shell: cmd run: | - set path=%path%;${{ env.OPENVINO_REPO }}\temp\tbb\bin + set path=%path%;${{ env.OPENVINO_REPO }}\temp\tbb\bin;${{ env.BUILD_DIR }}\bin\Release ctest -C ${{ env.CMAKE_BUILD_TYPE }} --test-dir ${{ env.BUILD_DIR }} -V -L UNIT - name: Perform code tracing via ITT collector shell: cmd run: | - set path=%path%;${{ env.OPENVINO_REPO }}\temp\tbb\bin + set path=%path%;${{ env.OPENVINO_REPO }}\temp\tbb\bin;${{ env.BUILD_DIR }}\bin\Release python3 ${{ env.OPENVINO_REPO }}\thirdparty\itt_collector\runtool\sea_runtool.py ^ --bindir ${{ env.OPENVINO_REPO }}\bin\intel64\${{ env.CMAKE_BUILD_TYPE }} ^ @@ -233,7 +233,7 @@ jobs: Compress-Archive @compress $compress = @{ - Path = "${{ env.OPENVINO_REPO }}/bin/intel64/${{ env.CMAKE_BUILD_TYPE }}/ov_cpu_func_tests.exe", "${{ env.OPENVINO_REPO }}/bin/intel64/${{ env.CMAKE_BUILD_TYPE }}/openvino_template_extension.dll", "${{ env.OPENVINO_REPO }}/src/tests/test_utils/functional_test_utils/layer_tests_summary", "${{ env.INSTALL_DIR }}/runtime/3rdparty/tbb" + Path = "${{ env.OPENVINO_REPO }}/bin/intel64/${{ env.CMAKE_BUILD_TYPE }}/ov_cpu_func_tests.exe", "${{ env.BUILD_DIR }}/bin/Release/ze_loader.dll", "${{ env.OPENVINO_REPO }}/bin/intel64/${{ env.CMAKE_BUILD_TYPE }}/openvino_template_extension.dll", "${{ env.OPENVINO_REPO }}/src/tests/test_utils/functional_test_utils/layer_tests_summary", "${{ env.INSTALL_DIR }}/runtime/3rdparty/tbb" CompressionLevel = "Optimal" DestinationPath = "${{ env.BUILD_DIR }}/openvino_tests.zip" } @@ -356,7 +356,7 @@ jobs: - name: Run with CC-ed runtime shell: cmd run: | - set path=%path%;${{ env.OPENVINO_REPO }}\temp\tbb\bin + set path=%path%;${{ env.OPENVINO_REPO }}\temp\tbb\bin;${{ env.BUILD_DIR }}\bin\Release ${{ env.OPENVINO_REPO }}\bin\intel64\${{ env.CMAKE_BUILD_TYPE }}\benchmark_app.exe -niter 1 -nireq 1 -m ${{ env.MODELS_PATH }}\models\test_model\test_model_fp32.xml -d CPU CPU_Functional_Tests: @@ -414,7 +414,7 @@ jobs: - name: Intel CPU plugin func tests (parallel) shell: cmd run: | - set path=%path%;${{ env.INSTALL_TEST_DIR }}\tbb\bin;${{ env.INSTALL_TEST_DIR }}\tbb + set path=%path%;${{ env.INSTALL_TEST_DIR }}\tbb\bin;${{ env.INSTALL_TEST_DIR }}\tbb;${{ env.INSTALL_TEST_DIR }} python3 ${{ env.PARALLEL_TEST_SCRIPT }} -e ${{ env.INSTALL_TEST_DIR }}\ov_cpu_func_tests.exe -w ${{ env.INSTALL_TEST_DIR }} -s suite -rf 0 -- --gtest_print_time=1 --gtest_filter=*smoke* timeout-minutes: 60 diff --git a/.gitmodules b/.gitmodules index e275bd58cee..b483bfe8e9e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -72,3 +72,9 @@ [submodule "src/plugins/intel_cpu/thirdparty/mlas"] path = src/plugins/intel_cpu/thirdparty/mlas url = https://github.com/openvinotoolkit/mlas.git +[submodule "src/plugins/intel_npu/thirdparty/level-zero"] + path = src/plugins/intel_npu/thirdparty/level-zero + url = https://github.com/oneapi-src/level-zero.git +[submodule "src/plugins/intel_npu/thirdparty/level-zero-ext"] + path = src/plugins/intel_npu/thirdparty/level-zero-ext + url = https://github.com/intel/level-zero-npu-extensions.git diff --git a/cmake/features.cmake b/cmake/features.cmake index 4687016191a..5b53c1a4351 100644 --- a/cmake/features.cmake +++ b/cmake/features.cmake @@ -44,7 +44,16 @@ endif() ov_dependent_option (ENABLE_ONEDNN_FOR_GPU "Enable oneDNN with GPU support" ${ENABLE_ONEDNN_FOR_GPU_DEFAULT} "ENABLE_INTEL_GPU" OFF) +if(X86_64) + set(ENABLE_INTEL_NPU_DEFAULT ON) +else() + set(ENABLE_INTEL_NPU_DEFAULT OFF) +endif() + +ov_dependent_option (ENABLE_INTEL_NPU "NPU plugin for OpenVINO runtime" ${ENABLE_INTEL_NPU_DEFAULT} "X86 OR X86_64;NOT APPLE" OFF) + ov_option (ENABLE_DEBUG_CAPS "enable OpenVINO debug capabilities at runtime" OFF) +ov_dependent_option (ENABLE_NPU_DEBUG_CAPS "enable NPU debug capabilities at runtime" ON "ENABLE_DEBUG_CAPS;ENABLE_INTEL_NPU" OFF) ov_dependent_option (ENABLE_GPU_DEBUG_CAPS "enable GPU debug capabilities at runtime" ON "ENABLE_DEBUG_CAPS;ENABLE_INTEL_GPU" OFF) ov_dependent_option (ENABLE_CPU_DEBUG_CAPS "enable CPU debug capabilities at runtime" ON "ENABLE_DEBUG_CAPS;ENABLE_INTEL_CPU" OFF) ov_dependent_option (ENABLE_SNIPPETS_DEBUG_CAPS "enable Snippets debug capabilities at runtime" ON "ENABLE_DEBUG_CAPS" OFF) diff --git a/cmake/packaging/debian.cmake b/cmake/packaging/debian.cmake index 905fba2118e..b44f031ce47 100644 --- a/cmake/packaging/debian.cmake +++ b/cmake/packaging/debian.cmake @@ -183,7 +183,7 @@ macro(ov_cpack_settings) endif() # intel-npu - if(ENABLE_INTEL_NPU OR BUILD_npu OR BUILD_vpux-plugin OR BUILD_applications.ai.vpu-accelerators.vpux-plugin) + if(ENABLE_INTEL_NPU) set(CPACK_COMPONENT_NPU_DESCRIPTION "Intel® Neural Processing Unit inference plugin") set(CPACK_COMPONENT_NPU_DEPENDS "${OV_CPACK_COMP_CORE}") set(CPACK_DEBIAN_NPU_PACKAGE_NAME "libopenvino-intel-npu-plugin-${cpack_name_ver}") diff --git a/cmake/packaging/rpm.cmake b/cmake/packaging/rpm.cmake index df5a30f6f54..71d51863268 100644 --- a/cmake/packaging/rpm.cmake +++ b/cmake/packaging/rpm.cmake @@ -39,6 +39,8 @@ macro(ov_cpack_settings) (NOT item MATCHES "^${OV_CPACK_COMP_PYTHON_OPENVINO_PACKAGE}_python.*" OR ENABLE_PYTHON_PACKAGING) AND # temporary block nvidia NOT item STREQUAL "nvidia" AND + # temporary block npu + NOT item STREQUAL "npu" AND # don't install Intel OpenMP NOT item STREQUAL "omp" AND # the same for pugixml @@ -179,7 +181,7 @@ macro(ov_cpack_settings) endif() # intel-npu - if(ENABLE_INTEL_NPU OR BUILD_npu OR BUILD_vpux-plugin OR BUILD_applications.ai.vpu-accelerators.vpux-plugin) + if(ENABLE_INTEL_NPU AND "npu" IN_LIST CPACK_COMPONENTS_ALL) set(CPACK_COMPONENT_NPU_DESCRIPTION "Intel® Neural Processing Unit inference plugin") set(CPACK_RPM_NPU_PACKAGE_REQUIRES "${core_package}") set(CPACK_RPM_NPU_PACKAGE_NAME "libopenvino-intel-npu-plugin-${cpack_name_ver}") diff --git a/src/inference/include/openvino/runtime/intel_npu/properties.hpp b/src/inference/include/openvino/runtime/intel_npu/properties.hpp new file mode 100644 index 00000000000..6eb47d32c86 --- /dev/null +++ b/src/inference/include/openvino/runtime/intel_npu/properties.hpp @@ -0,0 +1,57 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +/** + * @brief A header for advanced hardware related properties for NPU plugin + * To use in set_property, compile_model, import_model, get_property methods + * + * @file openvino/runtime/intel_npu/properties.hpp + */ +#pragma once + +#include "openvino/runtime/properties.hpp" + +namespace ov { + +/** + * @defgroup ov_runtime_npu_prop_cpp_api Intel NPU specific properties + * @ingroup ov_runtime_cpp_api + * Set of Intel NPU specific properties. + */ + +/** + * @brief Namespace with Intel NPU specific properties + */ +namespace intel_npu { + +/** + * @brief [Only for NPU plugin] + * Type: uint64_t + * Read-only property to get size of already allocated NPU DDR memory (both for discrete/integrated NPU devices) + * + * Note: Queries driver both for discrete/integrated NPU devices + * @ingroup ov_runtime_npu_prop_cpp_api + */ +static constexpr ov::Property device_alloc_mem_size{"NPU_DEVICE_ALLOC_MEM_SIZE"}; + +/** + * @brief [Only for NPU plugin] + * Type: uint64_t + * Read-only property to get size of available NPU DDR memory (both for discrete/integrated NPU devices) + * + * Note: Queries driver both for discrete/integrated NPU devices + * @ingroup ov_runtime_npu_prop_cpp_api + */ +static constexpr ov::Property device_total_mem_size{"NPU_DEVICE_TOTAL_MEM_SIZE"}; + +/** + * @brief [Only for NPU plugin] + * Type: uint32_t + * Read-only property to get NPU driver version (for both discrete/integrated NPU devices) + * @ingroup ov_runtime_npu_prop_cpp_api + */ +static constexpr ov::Property driver_version{"NPU_DRIVER_VERSION"}; + +} // namespace intel_npu +} // namespace ov diff --git a/src/plugins/intel_npu/CMakeLists.txt b/src/plugins/intel_npu/CMakeLists.txt new file mode 100644 index 00000000000..5e164845b00 --- /dev/null +++ b/src/plugins/intel_npu/CMakeLists.txt @@ -0,0 +1,29 @@ +# Copyright (C) 2018-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +if (NOT ENABLE_INTEL_NPU) + return() +endif() + +# +# Build properties +# + +set(NPU_DEVICE_NAME "NPU") +string(TOLOWER "${NPU_DEVICE_NAME}" NPU_PLUGIN_COMPONENT) +set(NPU_INTERNAL_COMPONENT "${NPU_PLUGIN_COMPONENT}_internal") + +set(NPU_PLUGIN_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + +include(cmake/features.cmake) + +set(CMAKE_CXX_STANDARD 17) + +if(ENABLE_NPU_DEBUG_CAPS) + add_compile_definitions(NPU_PLUGIN_DEVELOPER_BUILD) +endif() + +add_subdirectory(thirdparty EXCLUDE_FROM_ALL) + +add_subdirectory(src) diff --git a/src/plugins/intel_npu/README.md b/src/plugins/intel_npu/README.md new file mode 100644 index 00000000000..c4acbf168bc --- /dev/null +++ b/src/plugins/intel_npu/README.md @@ -0,0 +1,218 @@ +# NPU Plugin + +## Introduction   + +This is the OpenVINO Plugin for Intel® Neural Processing Unit (NPU) devices. + +  +## Supported Platforms + +OpenVINO™ toolkit is officially supported and validated on the following platforms: + +| Host | NPU device | OS (64-bit) | +| :--- | :--- | :--- | +| Raptor Lake (discrete NPU) | NPU 3700 | MS Windows* 11 | +| Meteor Lake (integrated NPU) | NPU 3720 | Ubuntu* 22, MS Windows* 11 | + + +  +## High Level Design + +![High Level Design](./docs/img/high_level_design.png) + + +  +## Description + +NPU Plugin is a software library that: +* Implements the unified OpenVINO Plugin API used to compile and execute neural networks on NPU devices. +* Uses the graph extension API exposed by the driver to convert the OpenVINO specific representation of the model into a proprietary format. The compiler performs platform specific optimizations in order to efficiently schedule the execution of layers and memory transactions on various NPU hardware submodules. +* Uses the Level Zero API implemented by the NPU user mode driver (UMD) to execute the model on the device. + +The plugin library is included inside the OpenVINO package while the compiler is packaged inside UMD and released separately. + +Note: Aligning with the platform and OpenVINO documentation, neural networks will be referred to with the more generic term of models in the rest of this document. + +  +## Model Compilation + +NPU plugin implements the OpenVINO Core "compile_model" API that converts the model representation into a proprietary format that can be executed on the NPU device: + +``` + ov::CompiledModel compiled_model = core.compile_model(model, "NPU" [, config]); +``` + +### Model caching + +There are two important compilation related metrics when executing models on NPU devices: +* First Ever Inference Latency (FEIL): Measures all steps required to compile and execute a model on the device for the first time. It includes model compilation time, the time required to load and initialize the model on the device and the first inference execution. +* First Inference Latency (FIL): Measures the time required to load and initialize the pre-compiled model on the device and the first inference execution. + + +#### UMD dynamic model caching + +UMD model caching is enabled by default in the current NPU driver to improve time to first inference (FIL). The model is stored in the cache after the compilation (included in FEIL) based on a hash key. The UMD generates the key from the input IR model and build arguments and then requests the DirectX Shader cache session to store the model with the computed key. Any subsequent request to compile the same IR model with the same arguments would cause the pre-compiled model to be read from the cache instead of being recompiled. + +#### OpenVINO model caching + +It is enabled when `ov::cache_dir` property is set and it is a common mechanism for all OpenVINO plugins. UMD model caching will be automatically bypassed by the NPU plugin when `ov::cache_dir` is set so the model will only be stored in the OpenVINO cache after the compilation. When a cache hit occurs for subsequent compilation requests, plugin will import the model instead of recompiling it. + +More details about OpenVINO model caching can be found here: [Model Caching Overview](https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_Model_caching_overview.html). + +### Compiler adapters + +Two additional layers are required to support the compiler from driver: +* Compiler Adapter - It serializes the OpenVINO internal representation of the model (ov::model) into an in-memory IR that will be provided to the NPU driver +* VCL - It deserializes the in-memory IR given by the NPU driver and prepares it for the compiler + +The interface between plugin and driver is based on an in-memory IR to facilitate backward and forward compatibility between two software packages (OpenVINO and NPU driver) that inherently have a different release cadence. + +  +## Model Execution + +NPU plugin will use the Level Zero (L0) API to execute the precompiled model on the NPU Device. The inference is executed as a standard L0 workload by describing the required tasks inside a command list and by submitting the list to the command queue for execution. The plugin will not use the CPU to execute any part of the inference workload. No pre/post processing workloads are executed on the CPU either, the entire inference will be offloaded on the NPU device. + +### Device management + +There is currently no support for multiple devices (N x discrete + integrated), one single level zero device will be enumerated during level zero backend initialization. Support for multiple devices will be added in future releases. + +### Inference pipeline + +The result of the model compilation is represented through a NetworkDescription. This model description is passed by the plugin to the driver to create a level zero graph instance and obtain a graph handle that can later be used to execute multiple inferences in parallel for the same model. Since the same model instance is shared across all subsequent inference objects, this initialization step is performed by default right after the model is compiled and it can be postponed until the creation of the first inference request through the use of an environment variable: "IE_NPU_CREATE_EXECUTOR" (IE_NPU_CREATE_EXECUTOR=0 to postpone the initialization). + +Users can create one or more inference requests for a compiled model using OpenVINO API: + +``` + ov::InferRequest request = compiled_model.create_infer_request(); +``` + +One unique level zero command queue is currently used to execute all inference requests created for the same model. + + +### Memory allocation + +Each inference request is linked to an already compiled model and maintains a reference to the common model description. The level zero (user mode) driver parses the model description and returns the number and size of input/output buffers that need to be allocated per inference request. NPU plugin allocates input/output buffers through dedicated level zero API. Pointers to input/output buffers need to be provided when the command list is created so a different command list is used for each inference request. + +The set of input and output buffers allocated by the NPU plugin for each inference request can be retrieved using the following API: +``` + ov::Tensor requestTensor = inferRequest.get_tensor(tensor_name); +``` + +Users can access the "data" member of the specified tensor and can directly populate it or read it. This is the recommended usage of input and output buffers that guarantees no extra copies will be performed by the NPU plugin. + +Alternatively, users can configure different input or output buffers to be used during inference: +``` + inferRequest.set_tensor(tensor_name, tensor); +``` + +Since these tensors are not natively accessible by the NPU device, plugin will perform the required memory copies to and from the original buffers that were allocated when the inference request was created. This has an impact on the inference latency and should be avoided whenever possible. + +Once the inference request is created and input/output tensors are prepared for the inference, the execution can be triggered either synchronously or asynchronously: +* Synchronous execution: +``` + inferRequest.infer(); +``` +* Asynchronous execution using: +``` + inferRequest.start_async(); + inferRequest.wait(); // optional, in case user callback is not provided +``` + +Multiple inferences can be executed in parallel, either from the same application thread through the use of asynchronous methods or from multiple threads with any of the available methods (synchronous or asynchronous). There is an optimal number of inference requests to be executed in parallel that would yield the best throughput without impacting the latency observed for each inference. This optimal number of requests is different for each model and depends on the ratio between the duration of the model execution on the DPU HW and the rest of the latency required to pass the request through the entire software stack. The NPU plugin returns the optimal number of inference requests through a dedicated property (`ov::optimal_number_of_infer_requests`). + +Note: the current implementation of this property does not estimate or check the duration of the model execution. A fixed number of recommended inference requests is currently returned based on the existing performance data gathered from a reduced set of models with different topologies. + +  +## Supported Properties + +Properties can be used to query and adjust the behavior of the NPU plugin itself or various parameters that control model compilation and execution. + +The following methods are made available to return the value of a given property (at core level or model specific): +``` + plugin_properties = ov.get_property("NPU", ); + [...] + model_properties = compiled_model.get_property(); +``` + +The following methods are made available to set the value of a given property (at core level or model specific): +``` + ov.set_property("NPU", {{Key, Value}}); + [...] + compiled_model.set_property({{Key, Value}}); +``` + +The following properties are supported: + +| Parameter Name | | Description | Supported Values | Default Value | +| :--- | :--- | :--- |:--- |:-- | +| `ov::supported_properties`/
`SUPPORTED_METRICS`/
`SUPPORTED_CONFIG_KEYS` | RO | Returns a list of all supported properties.
Can be queried on runtime. | `N/A` | `N/A` | +| `ov::caching_properties`/
`CACHING_PROPERTIES` | RW | Returns a list of all properties that are used by OpenVINO cache to build the hash key. | `N/A` | `N/A` | +| `ov::compilation_num_threads`/
`COMPILATION_NUM_THREADS` | RW | Maximum number of threads that can be used for compilation tasks. | `N/A` | `N/A` | +| `ov::num_streams`/
`NUM_STREAMS` | RO | Not used by the NPU plugin.
Always set to 1. | `AUTO/`
`INT` | `1` | +| `ov::optimal_number_of_infer_requests`/
`OPTIMAL_NUMBER_OF_INFER_REQUESTS` | RO | Returns the optimal number of inference requests to be used by the application. Depends on the platform version and on ov::hint::performance_mode. Please see the table below. | `N/A` | `N/A` | +| `ov::range_for_async_infer_requests`/
`RANGE_FOR_ASYNC_INFER_REQUESTS` | RO | Returns a tuple (bottom, top, step).
Not used by the NPU plugin. | `N/A` | `N/A` | +| `ov::range_for_streams`/
`RANGE_FOR_STREAMS` | RO | Returns a tuple (bottom, top).
Not used by the NPU plugin. | `N/A`| `N/A` | +| `ov::enable_profiling`/
`PERF_COUNT` | RW | Enables or disables performance counters. | `YES`/ `NO` | `NO` | +| `ov::hint::performance_mode`/
`PERFORMANCE_HINT` | RW | Sets the performance profile used to determine default values of DPUs/DMAs/NIREQs.
Default values for each profile are documented below. | `THROUGHPUT`/
`LATENCY`/
`UNDEFINED` | `UNDEFINED` | +| `ov::hint::num_requests`/
`PERFORMANCE_HINT_NUM_REQUESTS` | RW | Sets the number of outstanding inference requests. | `[0-]` | `1` | +| `ov::hint::model_priority`/
`MODEL_PRIORITY` | RW | Assigns a priority for the model execution. | `LOW`/
`MEDIUM`/
`HIGH` | `MEDIUM` | +| `ov::hint::enable_cpu_pinning`/
`ENABLE_CPU_PINNING` | RW | Allows CPU threads pinning during inference. | `YES`/ `NO` /
`NO` +| `ov::log::level`/
`LOG_LEVEL` | RW | Sets the log level for NPU Plugin. An environment variable is also made available to expose logs from early initialization phase: OV_NPU_LOG_LEVEL. | `LOG_LEVEL_NONE`/
`LOG_LEVEL_ERROR`/
`LOG_LEVEL_WARNING`/
`LOG_LEVEL_DEBUG`/
`LOG_LEVEL_TRACE` | `_NONE` | +| `ov::cache_dir`/
`CACHE_DIR` | RW | Folder path to be used by the OpenVINO cache. | `N/A` | empty | +| `ov::available_devices`/
`AVAILABLE_DEVICES` | RO | Returns the list of enumerated NPU devices.
NPU plugin does not currently support multiple devices. | `N/A`| `N/A` | +| `ov::device::id`/
`DEVICE_ID` | RW | Device identifier. Empty means auto detection. | empty/
`3700`/
`3720` | empty | +| `ov::device::uuid`/
| RO | Returns the Universal Unique ID of the NPU device. | `N/A`| `N/A` | +| `ov::device::architecture`/
`DEVICE_ARCHITECTURE` | RO | Returns the platform information. | `N/A`| `N/A` | +| `ov::device::full_name`/
`FULL_DEVICE_NAME` | RO | Returns the full name of the NPU device. | `N/A`| `N/A` | +| `ov::internal::exclusive_async_requests`/
`EXCLUSIVE_ASYNC_REQUESTS` | RW | Allows to use exclusive task executor for asynchronous infer requests. | `YES`/ `NO`| `NO` | + +  +### Performance Hint: Default Number of DPU Groups / DMA Engines + +The following table shows the default values for the number of DPU Groups (Tiles) and DMA Engines selected by the plugin based on the performance mode (THROUGHPUT/LATENCY) and based on the platform: + +| Performance hint | NPU Platform | Number of DPU Groups | Number of DMA Engines | +| :--- | :--- | :--- | :--- | +| THROUGHPUT | 3700 | 1 | 1 | +| THROUGHPUT | 3720 | 2 (all of them) | 2 (all of them) | +| LATENCY | 3700 | 4 (all of them) | 1 | +| LATENCY | 3720 | 2 (all of them) | 2 (all of them) | + +  +### Performance Hint: Optimal Number of Inference Requests + +The following table shows the optimal number of inference requests returned by the plugin based on the performance mode (THROUGHPUT/LATENCY) and based on the platform: + +| NPU Platform | Nr. of Inference Requests
THROUGHPUT | Nr. of Inference Requests
LATENCY | +| :--- | :--- | :--- | +| 3700 | 8 | 1 | +| 3720 | 4 | 1 | + + +  +## Stateful models + +Key ingredients to support stateful models which distinguish them from other models are: +* Implementing ReadValue and Assign operators +* Implementing Query State API (to give user an API to reset/get/set states) +* Implementing initialization for a state + +More details on how OpenVINO supports stateful models can be found here: [Stateful models](https://docs.openvino.ai/2022.3/openvino_docs_OV_UG_network_state_intro.html). + +The current implementation of state variables inside the NPU plugin is illustrated by the below diagram: + +![High Level Design](./docs/img/stateful_models.png) + +Notes on the implementation: +* One network with N inputs + K state variables + M outputs will be converted by the compiler into a model with (N+K) inputs and (M+K) outputs. State variables are represented by a set of input/output nodes. This is currently needed because the underlying software stack (driver and runtime) does not support state variables. +* The input and output nodes corresponding to state variables have different buffers allocated through the Level Zero API. +* The content of the output buffer is copied back into the input buffer by the plugin through the use of an intermediate state buffer: + * NPU Plugin allocates and maintains one additional state buffer which is exposed through the GetState/SetState API + * The actual level zero input buffer for the state is updated when the inference is triggered with the content of the state buffer + * The state buffer is updated once the inference is completed with the content of the output level zero buffer + +The implementation of state variables in the NPU plugin will be improved for upcoming releases. + +  +## Dynamic shapes +Dynamic shapes are not supported by the NPU plugin yet. diff --git a/src/plugins/intel_npu/cmake/features.cmake b/src/plugins/intel_npu/cmake/features.cmake new file mode 100644 index 00000000000..1eb4b0de7d9 --- /dev/null +++ b/src/plugins/intel_npu/cmake/features.cmake @@ -0,0 +1,18 @@ +# Copyright (C) 2018-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +ov_option(ENABLE_MLIR_COMPILER "Enable compilation of npu_mlir_compiler libraries" ON) + +ov_option(BUILD_COMPILER_FOR_DRIVER "Enable build of npu_driver_compiler" OFF) + +ov_dependent_option(ENABLE_DRIVER_COMPILER_ADAPTER "Enable NPU Compiler inside driver" ON "NOT BUILD_COMPILER_FOR_DRIVER" OFF) + +if(NOT BUILD_SHARED_LIBS AND NOT ENABLE_MLIR_COMPILER AND NOT ENABLE_DRIVER_COMPILER_ADAPTER) + message(FATAL_ERROR "No compiler found for static build!") +endif() + +# if ENABLE_ZEROAPI_BACKEND=ON, it adds the ze_loader dependency for driver compiler +ov_dependent_option(ENABLE_ZEROAPI_BACKEND "Enable zero-api as a plugin backend" ON "NOT BUILD_COMPILER_FOR_DRIVER" OFF) + +ov_dependent_option(ENABLE_IMD_BACKEND "Enable InferenceManagerDemo based NPU AL backend" OFF "NOT WIN32;NOT CMAKE_CROSSCOMPILING" OFF) diff --git a/src/plugins/intel_npu/docs/img/high_level_design.png b/src/plugins/intel_npu/docs/img/high_level_design.png new file mode 100644 index 00000000000..d3dbaad1feb --- /dev/null +++ b/src/plugins/intel_npu/docs/img/high_level_design.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5de0598bb4f332011f2e7ed1dfc2c8ff535eb572c586bc1e1a9df63101c8499b +size 20299 diff --git a/src/plugins/intel_npu/docs/img/stateful_models.png b/src/plugins/intel_npu/docs/img/stateful_models.png new file mode 100644 index 00000000000..539e487dc3a --- /dev/null +++ b/src/plugins/intel_npu/docs/img/stateful_models.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e3b9dade2b9e6a29919174f2ab2c03f370c5832ae191657d6822a8966bbd55d +size 49296 diff --git a/src/plugins/intel_npu/src/CMakeLists.txt b/src/plugins/intel_npu/src/CMakeLists.txt new file mode 100644 index 00000000000..278504bba02 --- /dev/null +++ b/src/plugins/intel_npu/src/CMakeLists.txt @@ -0,0 +1,22 @@ +# Copyright (C) 2018-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +# Close LTO temporarily for add_symbol_to_partition_1 issue +set(ENABLE_LTO OFF) + +add_subdirectory(utils) + +add_subdirectory(al) + +if(ENABLE_DRIVER_COMPILER_ADAPTER) + add_subdirectory(compiler) +endif() + +if(ENABLE_ZEROAPI_BACKEND) + add_subdirectory(backend) +endif() + +if (NOT BUILD_COMPILER_FOR_DRIVER) + add_subdirectory(plugin) +endif() diff --git a/src/plugins/intel_npu/src/al/CMakeLists.txt b/src/plugins/intel_npu/src/al/CMakeLists.txt new file mode 100644 index 00000000000..b7eff57de31 --- /dev/null +++ b/src/plugins/intel_npu/src/al/CMakeLists.txt @@ -0,0 +1,39 @@ +# Copyright (C) 2018-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +set(TARGET_NAME openvino_npu_al) + +file(GLOB_RECURSE SOURCES *.cpp *.hpp *.h) +source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR} FILES ${SOURCES}) + +add_library(${TARGET_NAME} STATIC ${SOURCES}) +add_library(openvino::npu_al ALIAS ${TARGET_NAME}) +set_target_properties(${TARGET_NAME} PROPERTIES EXPORT_NAME npu_al) + +target_include_directories(${TARGET_NAME} + PUBLIC + $ +) + +if(ENABLE_MLIR_COMPILER) + target_compile_definitions(${TARGET_NAME} PRIVATE ENABLE_MLIR_COMPILER) +endif() + +target_link_libraries(${TARGET_NAME} + PUBLIC + openvino::npu_logger_utils + openvino::runtime::dev +) + +set_target_properties(${TARGET_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO}) +ov_add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME}) + +# +# targets install +# +ov_install_static_lib(${TARGET_NAME} ${NPU_PLUGIN_COMPONENT}) + +ov_developer_package_export_targets(TARGET openvino::npu_al + INSTALL_INCLUDE_DIRECTORIES + ${CMAKE_CURRENT_SOURCE_DIR}/include) diff --git a/src/plugins/intel_npu/src/al/include/device_helpers.hpp b/src/plugins/intel_npu/src/al/include/device_helpers.hpp new file mode 100644 index 00000000000..43b9ccf93ce --- /dev/null +++ b/src/plugins/intel_npu/src/al/include/device_helpers.hpp @@ -0,0 +1,16 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +#include "npu_private_properties.hpp" + +namespace utils { +bool isNPUDevice(const uint32_t deviceId); +uint32_t getSliceIdBySwDeviceId(const uint32_t swDevId); +std::string getPlatformByDeviceName(const std::string& deviceName); +} // namespace utils diff --git a/src/plugins/intel_npu/src/al/include/intel_npu/al/config/common.hpp b/src/plugins/intel_npu/src/al/include/intel_npu/al/config/common.hpp new file mode 100644 index 00000000000..4e8285e6c5d --- /dev/null +++ b/src/plugins/intel_npu/src/al/include/intel_npu/al/config/common.hpp @@ -0,0 +1,249 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "intel_npu/al/config/config.hpp" +#include "npu_private_properties.hpp" +#include "openvino/runtime/internal_properties.hpp" +#include "openvino/runtime/properties.hpp" + +namespace intel_npu { + +// +// register +// + +void registerCommonOptions(OptionsDesc& desc); + +// +// PERFORMANCE_HINT +// + +struct PERFORMANCE_HINT final : OptionBase { + static std::string_view key() { + return ov::hint::performance_mode.name(); + } + + static constexpr std::string_view getTypeName() { + return "ov::hint::PerformanceMode"; + } + + static ov::hint::PerformanceMode defaultValue() { + return ov::hint::PerformanceMode::LATENCY; + } + + static ov::hint::PerformanceMode parse(std::string_view val); +}; + +// PERFORMANCE_HINT_NUM_REQUESTS +// + +struct PERFORMANCE_HINT_NUM_REQUESTS final : OptionBase { + static std::string_view key() { + return ov::hint::num_requests.name(); + } + + /** + * @brief Returns configuration value if it is valid, otherwise throws + * @details This is the same function as "InferenceEngine::PerfHintsConfig::CheckPerformanceHintRequestValue", + * slightly modified as to not rely on the legacy API anymore. + * @param configuration value as string + * @return configuration value as number + */ + static uint32_t parse(std::string_view val) { + int val_i = -1; + try { + val_i = std::stoi(val.data()); + if (val_i >= 0) + return val_i; + else + throw std::logic_error("wrong val"); + } catch (const std::exception&) { + OPENVINO_THROW("Wrong value of ", + val.data(), + " for property key ", + ov::hint::num_requests.name(), + ". Expected only positive integer numbers"); + } + } + + static uint32_t defaultValue() { + // Default value depends on PERFORMANCE_HINT, see getOptimalNumberOfInferRequestsInParallel + // 1 corresponds to LATENCY and default mode (hints not specified) + return 1u; + } +}; + +// +// INFERENCE_PRECISION_HINT +// + +struct INFERENCE_PRECISION_HINT final : OptionBase { + static std::string_view key() { + return ov::hint::inference_precision.name(); + } + + static constexpr std::string_view getTypeName() { + return "ov::hint::inference_precision"; + } + + static ov::element::Type defaultValue() { + return ov::element::f16; + } + + static ov::element::Type parse(std::string_view val) { + if (val.empty() || (val == "f16")) { + return ov::element::f16; + } else if (val == "i8") { + return ov::element::i8; + } else { + OPENVINO_THROW("Wrong value ", + val.data(), + " for property key ", + ov::hint::inference_precision.name(), + ". Supported values: f16, i8"); + } + }; +}; + +// +// PERF_COUNT +// + +struct PERF_COUNT final : OptionBase { + static std::string_view key() { + return ov::enable_profiling.name(); + } + + static bool defaultValue() { + return false; + } +}; + +// +// LOG_LEVEL +// + +struct LOG_LEVEL final : OptionBase { + static std::string_view key() { + return ov::log::level.name(); + } + + static constexpr std::string_view getTypeName() { + return "ov::log::Level"; + } + + static std::string_view envVar() { + return "OV_NPU_LOG_LEVEL"; + } + + static ov::log::Level defaultValue() { + return ov::log::Level::NO; + } +}; + +// +// PLATFORM +// + +struct PLATFORM final : OptionBase { + static std::string_view key() { + return ov::intel_npu::platform.name(); + } + + static std::string defaultValue() { + return std::string(ov::intel_npu::Platform::AUTO_DETECT); + } + +#ifdef NPU_PLUGIN_DEVELOPER_BUILD + static std::string_view envVar() { + return "IE_NPU_PLATFORM"; + } +#endif + + static bool isPublic() { + return false; + } +}; + +// +// DEVICE_ID +// + +struct DEVICE_ID final : OptionBase { + static std::string_view key() { + return ov::device::id.name(); + } + + static std::string defaultValue() { + return {}; + } +}; + +// +// CACHE_DIR +// + +struct CACHE_DIR final : OptionBase { + static std::string_view key() { + return ov::cache_dir.name(); + } + + static std::string defaultValue() { + return {}; + } +}; + +// +// LOADED_FROM_CACHE +// + +struct LOADED_FROM_CACHE final : OptionBase { + static std::string_view key() { + return ov::loaded_from_cache.name(); + } + + static bool defaultValue() { + return false; + } +}; + +// +// CACHING PROPERTIES +// + +struct CACHING_PROPERTIES final : OptionBase { + static std::string_view key() { + return ov::internal::caching_properties.name(); + } + + static std::string defaultValue() { + return {}; + } +}; + +// +// INTERNAL SUPPORTED PROPERTIES +// + +struct INTERNAL_SUPPORTED_PROPERTIES final : OptionBase { + static std::string_view key() { + return ov::internal::supported_properties.name(); + } + + static std::string defaultValue() { + return {}; + } +}; + +} // namespace intel_npu + +namespace ov { +namespace hint { + +std::string_view stringifyEnum(PerformanceMode val); + +} // namespace hint +} // namespace ov diff --git a/src/plugins/intel_npu/src/al/include/intel_npu/al/config/compiler.hpp b/src/plugins/intel_npu/src/al/include/intel_npu/al/config/compiler.hpp new file mode 100644 index 00000000000..7259b996a4f --- /dev/null +++ b/src/plugins/intel_npu/src/al/include/intel_npu/al/config/compiler.hpp @@ -0,0 +1,363 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include "common.hpp" +#include "intel_npu/al/config/config.hpp" +#include "npu_private_properties.hpp" +#include "openvino/runtime/intel_npu/properties.hpp" + +namespace ov { + +namespace intel_npu { + +std::string_view stringifyEnum(CompilerType val); +std::string_view stringifyEnum(ElfCompilerBackend val); + +} // namespace intel_npu + +} // namespace ov + +namespace intel_npu { + +// +// register +// + +void registerCompilerOptions(OptionsDesc& desc); + +// +// COMPILER_TYPE +// + +struct COMPILER_TYPE final : OptionBase { + static std::string_view key() { + return ov::intel_npu::compiler_type.name(); + } + + static constexpr std::string_view getTypeName() { + return "ov::intel_npu::CompilerType"; + } + + static std::string_view envVar(); + + static ov::intel_npu::CompilerType defaultValue(); + + static ov::intel_npu::CompilerType parse(std::string_view val); + + static std::string toString(const ov::intel_npu::CompilerType& val); + + static OptionMode mode() { + return OptionMode::CompileTime; + } + + static bool isPublic() { + return false; + } +}; + +// +// COMPILATION_MODE +// + +struct COMPILATION_MODE final : OptionBase { + static std::string_view key() { + return ov::intel_npu::compilation_mode.name(); + } + +#ifdef NPU_PLUGIN_DEVELOPER_BUILD + static std::string_view envVar() { + return "IE_NPU_COMPILATION_MODE"; + } +#endif + + static std::string defaultValue() { + return ""; + } + + static OptionMode mode() { + return OptionMode::CompileTime; + } + + static bool isPublic() { + return false; + } +}; + +// +// DYNAMIC_SHAPE_TO_STATIC +// + +struct DYNAMIC_SHAPE_TO_STATIC final : OptionBase { + static std::string_view key() { + return ov::intel_npu::dynamic_shape_to_static.name(); + } + +#ifdef NPU_PLUGIN_DEVELOPER_BUILD + static std::string_view envVar() { + return "IE_NPU_DYNAMIC_SHAPE_TO_STATIC"; + } +#endif + + static bool defaultValue() { + return false; + } + + static OptionMode mode() { + return OptionMode::CompileTime; + } +}; + +// +// COMPILATION_MODE_PARAMS +// + +struct COMPILATION_MODE_PARAMS final : OptionBase { + static std::string_view key() { + return ov::intel_npu::compilation_mode_params.name(); + } + + static std::string defaultValue() { + return {}; + } + + static OptionMode mode() { + return OptionMode::CompileTime; + } + + static bool isPublic() { + return false; + } +}; + +// +// DPU_GROUPS +// + +struct DPU_GROUPS final : OptionBase { + static std::string_view key() { + return ov::intel_npu::dpu_groups.name(); + } + + static std::vector deprecatedKeys() { + return {}; + } + + static int64_t defaultValue() { + return -1; + } + + static OptionMode mode() { + return OptionMode::CompileTime; + } + + static bool isPublic() { + return false; + } + +#ifdef NPU_PLUGIN_DEVELOPER_BUILD + static std::string_view envVar() { + return "IE_NPU_DPU_GROUPS"; + } +#endif +}; + +// +// SELECTED_TILES +// + +struct TILES final : OptionBase { + static std::string_view key() { + return ov::intel_npu::tiles.name(); + } + + static std::vector deprecatedKeys() { + return {}; + } + + static int64_t defaultValue() { + return -1; + } + + static OptionMode mode() { + return OptionMode::CompileTime; + } + + static bool isPublic() { + return false; + } + +#ifdef NPU_PLUGIN_DEVELOPER_BUILD + static std::string_view envVar() { + return "IE_NPU_TILES"; + } +#endif +}; + +// +// STEPPING +// + +struct STEPPING final : OptionBase { + static std::string_view key() { + return ov::intel_npu::stepping.name(); + } + + static std::vector deprecatedKeys() { + return {}; + } + + static int64_t defaultValue() { + return -1; + } + + static OptionMode mode() { + return OptionMode::CompileTime; + } + + static bool isPublic() { + return false; + } +}; + +// +// MAX_TILES +// + +struct MAX_TILES final : OptionBase { + static std::string_view key() { + return ov::intel_npu::max_tiles.name(); + } + + static std::vector deprecatedKeys() { + return {}; + } + + static int64_t defaultValue() { + return -1; + } + + static OptionMode mode() { + return OptionMode::CompileTime; + } + + static bool isPublic() { + return false; + } +}; + +// +// DMA_ENGINES +// + +struct DMA_ENGINES final : OptionBase { + static std::string_view key() { + return ov::intel_npu::dma_engines.name(); + } + + static std::vector deprecatedKeys() { + return {}; + } + + static int64_t defaultValue() { + return -1; + } + + static OptionMode mode() { + return OptionMode::CompileTime; + } + + static bool isPublic() { + return false; + } + +#ifdef NPU_PLUGIN_DEVELOPER_BUILD + static std::string_view envVar() { + return "IE_NPU_DMA_ENGINES"; + } +#endif +}; + +// +// USE_ELF_COMPILER_BACKEND +// + +struct USE_ELF_COMPILER_BACKEND final : OptionBase { + static std::string_view key() { + return ov::intel_npu::use_elf_compiler_backend.name(); + } + + static constexpr std::string_view getTypeName() { + return "ov::intel_npu::ElfCompilerBackend"; + } + +#ifdef NPU_PLUGIN_DEVELOPER_BUILD + static std::string_view envVar() { + return "IE_NPU_USE_ELF_COMPILER_BACKEND"; + } +#endif + + static ov::intel_npu::ElfCompilerBackend defaultValue() { + return ov::intel_npu::ElfCompilerBackend::AUTO; + } + + static ov::intel_npu::ElfCompilerBackend parse(std::string_view val); + + static std::string toString(const ov::intel_npu::ElfCompilerBackend& val); +}; + +// +// BACKEND_COMPILATION_PARAMS +// + +struct BACKEND_COMPILATION_PARAMS final : OptionBase { + static std::string_view key() { + return ov::intel_npu::backend_compilation_params.name(); + } + + static std::string defaultValue() { + return {}; + } + + static OptionMode mode() { + return OptionMode::CompileTime; + } + + static bool isPublic() { + return false; + } +}; + +// +// COMPILATION_NUM_THREADS +// + +struct COMPILATION_NUM_THREADS final : OptionBase { + static std::string_view key() { + return ov::compilation_num_threads.name(); + } + + static int32_t defaultValue() { + return std::max(1, static_cast(std::thread::hardware_concurrency())); + } + + static void validateValue(const int32_t& num) { + if (num <= 0) { + OPENVINO_THROW("ov::compilation_num_threads must be positive int32 value"); + } + } + + static OptionMode mode() { + return OptionMode::CompileTime; + } + + static bool isPublic() { + return false; + } +}; + +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/al/include/intel_npu/al/config/config.hpp b/src/plugins/intel_npu/src/al/include/intel_npu/al/config/config.hpp new file mode 100644 index 00000000000..06cf7b314ac --- /dev/null +++ b/src/plugins/intel_npu/src/al/include/intel_npu/al/config/config.hpp @@ -0,0 +1,444 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "intel_npu/utils/logger/logger.hpp" +#include "openvino/core/except.hpp" +#include "openvino/runtime/properties.hpp" + +namespace intel_npu { + +template +struct TypePrinter { + static constexpr bool hasName() { + return false; + } + static constexpr const char* name(); +}; + +#define TYPE_PRINTER(type) \ + template <> \ + struct TypePrinter { \ + static constexpr bool hasName() { \ + return true; \ + } \ + static constexpr const char* name() { \ + return #type; \ + } \ + }; + +TYPE_PRINTER(bool) +TYPE_PRINTER(char) +TYPE_PRINTER(char*) +TYPE_PRINTER(int) +TYPE_PRINTER(unsigned int) +TYPE_PRINTER(int64_t) +TYPE_PRINTER(double) +TYPE_PRINTER(std::string) + +// +// OptionParser +// + +template +struct OptionParser; + +template <> +struct OptionParser final { + static std::string parse(std::string_view val) { + return {val.data(), val.size()}; + } +}; + +template <> +struct OptionParser final { + static bool parse(std::string_view val); +}; + +template <> +struct OptionParser final { + static int32_t parse(std::string_view val); +}; + +template <> +struct OptionParser final { + static int64_t parse(std::string_view val); +}; + +template <> +struct OptionParser final { + static uint64_t parse(std::string_view val); +}; + +template <> +struct OptionParser final { + static double parse(std::string_view val); +}; + +template <> +struct OptionParser final { + static ov::log::Level parse(std::string_view val); +}; + +void splitAndApply(const std::string& str, char delim, std::function callback); + +template +struct OptionParser> final { + static std::vector parse(std::string_view val) { + std::vector res; + splitAndApply(val, ',', [&](std::string_view item) { + res.push_back(OptionParser::parse(item)); + }); + return res; + } +}; + +template +struct OptionParser> final { + static std::chrono::duration parse(std::string_view val) { + std::istringstream stream(val.data()); + + Rep count{}; + if (stream >> count) { + OPENVINO_ASSERT(count >= 0, + "Value '", + count, + "' is not a valid time duration, non-negative values expected"); + return std::chrono::duration(count); + } + + OPENVINO_THROW("Can't parse '", val.data(), "' as time duration"); + } +}; + +// +// OptionPrinter +// + +template +struct OptionPrinter final { + static std::string toString(const T& val) { + std::stringstream ss; + if constexpr (std::is_floating_point_v>) { + ss << std::fixed << std::setprecision(2) << val; + } else if constexpr (std::is_enum_v>) { + ss << stringifyEnum(val); + return ss.str(); + } else { + ss << val; + } + return ss.str(); + } +}; + +// NB: boolean config option has values YES for true, NO for false +template <> +struct OptionPrinter final { + static std::string toString(bool val); +}; + +template +struct OptionPrinter> final { + static std::string toString(const std::chrono::duration& val) { + return std::to_string(val.count()); + } +}; + +template <> +struct OptionPrinter final { + static std::string toString(ov::log::Level val); +}; + +// +// OptionMode +// + +enum class OptionMode { + Both, + CompileTime, + RunTime, +}; + +std::string_view stringifyEnum(OptionMode val); + +// +// OptionBase +// + +// Actual Option description must inherit this class and pass itself as template parameter. +template +struct OptionBase { + using ValueType = T; + + // `ActualOpt` must implement the following method: + // static std::string_view key() + + static constexpr std::string_view getTypeName() { + if constexpr (TypePrinter::hasName()) { + return TypePrinter::name(); + } + static_assert(TypePrinter::hasName(), + "Options type is not a standard type, please add `getTypeName()` to your option"); + } + // Overload this to provide environment variable support. + static std::string_view envVar() { + return ""; + } + + // Overload this to provide deprecated keys names. + static std::vector deprecatedKeys() { + return {}; + } + + // Overload this to provide default value if it wasn't specified by user. + // If it is std::nullopt - exception will be thrown in case of missing option access. + static std::optional defaultValue() { + return std::nullopt; + } + + // Overload this to provide more specific parser. + static ValueType parse(std::string_view val) { + return OptionParser::parse(val); + } + + // Overload this to provide more specific validation + static void validateValue(const ValueType&) {} + + // Overload this to provide more specific implementation. + static OptionMode mode() { + return OptionMode::Both; + } + + // Overload this for private options. + static bool isPublic() { + return true; + } + + static std::string toString(const ValueType& val) { + return OptionPrinter::toString(val); + } +}; + +// +// OptionValue +// + +namespace details { + +class OptionValue { +public: + virtual ~OptionValue(); + + virtual std::string_view getTypeName() const = 0; + virtual std::string toString() const = 0; +}; + +template +class OptionValueImpl final : public OptionValue { + using ToStringFunc = std::string (*)(const T&); + +public: + template + OptionValueImpl(U&& val, ToStringFunc toStringImpl) : _val(std::forward(val)), + _toStringImpl(toStringImpl) {} + + std::string_view getTypeName() const override final { + if constexpr (TypePrinter::hasName()) { + return TypePrinter::name(); + } else { + return Opt::getTypeName(); + } + } + + const T& getValue() const { + return _val; + } + + std::string toString() const override { + return _toStringImpl(_val); + } + +private: + T _val; + ToStringFunc _toStringImpl = nullptr; +}; + +} // namespace details + +// +// OptionConcept +// + +namespace details { + +struct OptionConcept final { + std::string_view (*key)() = nullptr; + std::string_view (*envVar)() = nullptr; + OptionMode (*mode)() = nullptr; + bool (*isPublic)() = nullptr; + std::shared_ptr (*validateAndParse)(std::string_view val) = nullptr; +}; + +template +std::shared_ptr validateAndParse(std::string_view val) { + using ValueType = typename Opt::ValueType; + + try { + auto parsedVal = Opt::parse(val); + Opt::validateValue(parsedVal); + return std::make_shared>(std::move(parsedVal), &Opt::toString); + } catch (const std::exception& e) { + OPENVINO_THROW("Failed to parse '", Opt::key().data(), "' option : ", e.what()); + } +} + +template +OptionConcept makeOptionModel() { + return {&Opt::key, &Opt::envVar, &Opt::mode, &Opt::isPublic, &validateAndParse}; +} + +} // namespace details + +// +// OptionsDesc +// + +class OptionsDesc final { +public: + template + void add(); + + std::vector getSupported(bool includePrivate = false) const; + + details::OptionConcept get(std::string_view key, OptionMode mode) const; + void walk(std::function cb) const; + +private: + std::unordered_map _impl; + std::unordered_map _deprecated; +}; + +template +void OptionsDesc::add() { + OPENVINO_ASSERT(_impl.count(Opt::key().data()) == 0, "Option '", Opt::key().data(), "' was already registered"); + _impl.insert({Opt::key().data(), details::makeOptionModel()}); + + for (const auto& deprecatedKey : Opt::deprecatedKeys()) { + OPENVINO_ASSERT(_deprecated.count(deprecatedKey.data()) == 0, + "Option '", + deprecatedKey.data(), + "' was already registered"); + _deprecated.insert({deprecatedKey.data(), Opt::key().data()}); + } +} + +// +// Config +// + +class Config final { +public: + using ConfigMap = std::map; + using ImplMap = std::unordered_map>; + + explicit Config(const std::shared_ptr& desc); + + void update(const ConfigMap& options, OptionMode mode = OptionMode::Both); + + void parseEnvVars(); + + template + bool has() const; + + template + typename Opt::ValueType get() const; + + template + typename std::string getString() const; + + std::string toString() const; + +private: + std::shared_ptr _desc; + ImplMap _impl; +}; + +template +bool Config::has() const { + return _impl.count(Opt::key().data()) != 0; +} + +template +typename Opt::ValueType Config::get() const { + using ValueType = typename Opt::ValueType; + + auto log = Logger::global().clone("Config"); + log.trace("Get value for the option '%s'", Opt::key().data()); + + const auto it = _impl.find(Opt::key().data()); + + if (it == _impl.end()) { + const std::optional optional = Opt::defaultValue(); + log.trace("The option '%s' was not set by user, try default value", Opt::key().data()); + + OPENVINO_ASSERT(optional.has_value(), + "Option '", + Opt::key().data(), + "' was not provided, no default value is available"); + return optional.value(); + } + + OPENVINO_ASSERT(it->second != nullptr, "Got NULL OptionValue for :", Opt::key().data()); + + const auto optVal = std::dynamic_pointer_cast>(it->second); +#if defined(__CHROMIUMOS__) + if (optVal == nullptr) { + if (Opt::getTypeName() == it->second->getTypeName()) { + const auto val = std::static_pointer_cast>(it->second); + return val->getValue(); + } + } +#endif + OPENVINO_ASSERT(optVal != nullptr, + "Option '", + Opt::key().data(), + "' has wrong parsed type: expected '", + Opt::getTypeName().data(), + "', got '", + it->second->getTypeName().data(), + "'"); + + return optVal->getValue(); +} + +template +typename std::string Config::getString() const { + typename Opt::ValueType value = Config::get(); + + return Opt::toString(value); +} + +// +// envVarStrToBool +// + +bool envVarStrToBool(const char* varName, const char* varValue); + +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/al/include/intel_npu/al/config/runtime.hpp b/src/plugins/intel_npu/src/al/include/intel_npu/al/config/runtime.hpp new file mode 100644 index 00000000000..aa855f188c0 --- /dev/null +++ b/src/plugins/intel_npu/src/al/include/intel_npu/al/config/runtime.hpp @@ -0,0 +1,185 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "intel_npu/al/config/config.hpp" +#include "npu_private_properties.hpp" +#include "openvino/runtime/intel_npu/properties.hpp" +#include "openvino/runtime/internal_properties.hpp" +#include "openvino/runtime/properties.hpp" + +namespace ov { + +namespace intel_npu { + +std::string_view stringifyEnum(ProfilingType val); + +} // namespace intel_npu + +} // namespace ov + +namespace intel_npu { + +// +// register +// + +void registerRunTimeOptions(OptionsDesc& desc); + +// +// EXCLUSIVE_ASYNC_REQUESTS +// + +struct EXCLUSIVE_ASYNC_REQUESTS final : OptionBase { + static std::string_view key() { + return ov::internal::exclusive_async_requests.name(); + } + + static bool defaultValue() { + return false; + } + + static constexpr std::string_view getTypeName() { + return "bool"; + } + + static OptionMode mode() { + return OptionMode::RunTime; + } +}; + +int64_t getOptimalNumberOfInferRequestsInParallel(const Config& config); + +/// +/// PROFILING TYPE +/// +struct PROFILING_TYPE final : OptionBase { + static std::string_view key() { + return ov::intel_npu::profiling_type.name(); + } + + static constexpr std::string_view getTypeName() { + return "ov::intel_npu::ProfilingType"; + } + + static ov::intel_npu::ProfilingType defaultValue() { + return ov::intel_npu::ProfilingType::MODEL; + } + + static ov::intel_npu::ProfilingType parse(std::string_view val); + + static std::string toString(const ov::intel_npu::ProfilingType& val); + + static OptionMode mode() { + return OptionMode::RunTime; + } +}; + +// +// MODEL_PRIORITY +// + +struct MODEL_PRIORITY final : OptionBase { + static std::string_view key() { + return ov::hint::model_priority.name(); + } + + static constexpr std::string_view getTypeName() { + return "ov::hint::Priority"; + } + + static ov::hint::Priority defaultValue() { + return ov::hint::Priority::MEDIUM; + } + + static ov::hint::Priority parse(std::string_view val); + + static std::string toString(const ov::hint::Priority& val); + + static OptionMode mode() { + return OptionMode::RunTime; + } +}; + +// +// CREATE_EXECUTOR +// + +struct CREATE_EXECUTOR final : OptionBase { + static std::string_view key() { + return ov::intel_npu::create_executor.name(); + } + + static int64_t defaultValue() { + return 1; + } + +#ifdef NPU_PLUGIN_DEVELOPER_BUILD + static std::string_view envVar() { + return "IE_NPU_CREATE_EXECUTOR"; + } +#endif + + static bool isPublic() { + return false; + } + + static OptionMode mode() { + return OptionMode::RunTime; + } +}; + +// +// NUM_STREAMS +// +struct NUM_STREAMS final : OptionBase { + static std::string_view key() { + return ov::num_streams.name(); + } + + static constexpr std::string_view getTypeName() { + return "ov::streams::Num"; + } + + const static ov::streams::Num defVal; + + // The only supported number for currently supported platforms. + // FIXME: update in the future + static ov::streams::Num defaultValue() { + return defVal; + } + + static ov::streams::Num parse(std::string_view val); + + static std::string toString(const ov::streams::Num& val); + + static void validateValue(const ov::streams::Num& num) { + if (defVal != num && ov::streams::AUTO != num) { + throw std::runtime_error("NUM_STREAMS can not be set"); + } + } + + static OptionMode mode() { + return OptionMode::RunTime; + } +}; + +// +// ENABLE_CPU_PINNING +// +struct ENABLE_CPU_PINNING final : OptionBase { + static std::string_view key() { + return ov::hint::enable_cpu_pinning.name(); + } + + static bool defaultValue() { + return false; + } + + static OptionMode mode() { + return OptionMode::RunTime; + } +}; +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/al/include/intel_npu/al/icompiled_model.hpp b/src/plugins/intel_npu/src/al/include/intel_npu/al/icompiled_model.hpp new file mode 100644 index 00000000000..6d362547785 --- /dev/null +++ b/src/plugins/intel_npu/src/al/include/intel_npu/al/icompiled_model.hpp @@ -0,0 +1,37 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +#include "intel_npu/al/config/common.hpp" +#include "intel_npu/al/icompiler.hpp" +#include "openvino/runtime/icompiled_model.hpp" + +namespace intel_npu { + +class ICompiledModel : public ov::ICompiledModel { +public: + using ov::ICompiledModel::ICompiledModel; + + virtual const std::shared_ptr& get_network_description() const = 0; + + virtual const Config& get_config() const = 0; + + // Compiler is used for post-processing profiling data when using PERF_COUNT property + virtual const ov::SoPtr& get_compiler() const = 0; + + const NetworkMetadata& get_network_metadata() const { + return get_network_description()->metadata; + } + +protected: + std::shared_ptr shared_from_this() const { + return std::dynamic_pointer_cast(ov::ICompiledModel::shared_from_this()); + } +}; + +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/al/include/intel_npu/al/icompiler.hpp b/src/plugins/intel_npu/src/al/include/intel_npu/al/icompiler.hpp new file mode 100644 index 00000000000..ee926032508 --- /dev/null +++ b/src/plugins/intel_npu/src/al/include/intel_npu/al/icompiler.hpp @@ -0,0 +1,141 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +// Compiler Interface + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "intel_npu/al/config/config.hpp" +#include "openvino/core/partial_shape.hpp" +#include "openvino/core/type/element_type.hpp" +#include "openvino/runtime/common.hpp" +#include "openvino/runtime/profiling_info.hpp" + +namespace intel_npu { + +/** + * @brief A helper structure used for storing the metadata found within the I/O nodes. + * @details The "legacyName" attribute holds the name most commonly used as map key for multiple structures. + * This value also corresponds to the identifier used by the OpenVINO 1.0 API. + * + * "originalShape" corresponds to the shape registered in the graph, while "transposedShape" holds the shape obtained + * upon applying a transposition corresponding to the legacy layout value. Use the "transposedShape" one if not sure + * which one you need. + */ +struct IONodeDescriptor { + std::string legacyName; + std::string currentNodeName; + std::unordered_set outputTensorNames; + ov::element::Type_t precision; + ov::PartialShape originalShape; + ov::PartialShape transposedShape; +}; + +/** + * @brief A helper map to represent descriptions for inputs and outputs + * of a network + */ +using IONodeDescriptorMap = std::unordered_map; + +struct NetworkMetadata final { + std::string name; + + std::vector inputNames; + std::vector outputNames; + std::vector stateNames; + + IONodeDescriptorMap parameters; + IONodeDescriptorMap results; + IONodeDescriptorMap states; + IONodeDescriptorMap profilingOutputs; + + std::unordered_map inputOrder; + std::unordered_map outputOrder; + + int numStreams = 1; +}; + +/** + * @struct NetworkDescription + * @brief The object returned by the compiler + * to provide such information about a network as description of inputs and outputs, + * name and compiled network in a format executable by device + */ +struct NetworkDescription final { + NetworkDescription(std::vector&& compiledNetwork, NetworkMetadata&& metadata) + : compiledNetwork(std::move(compiledNetwork)), + metadata(std::move(metadata)) {} + // Force move semantics to prevent blob copies + NetworkDescription(const NetworkDescription&) = delete; + NetworkDescription(NetworkDescription&&) = default; + NetworkDescription& operator=(const NetworkDescription&) = delete; + NetworkDescription& operator=(NetworkDescription&&) = default; + ~NetworkDescription() = default; + + std::vector compiledNetwork; + + NetworkMetadata metadata; +}; + +/** + * @interface ICompiler + * @brief An interface to be implemented by a concrete compiler to provide + * methods for preparing a network for execution on a NPU device + */ +class ICompiler : public std::enable_shared_from_this { +public: + /** + * @brief Returns the maximum OpenVino opset version supported by the compiler + * @return opset version e.g. 11 for opset11 + */ + virtual uint32_t getSupportedOpsetVersion() const = 0; + + /** + * @brief Transforms a network from the OpenVINO model representation to a format executable + * by a NPU device + * @param model a shared pointer to the OpenVINO model to be compiled + * @param config a reference to NPUConfig containing plugin config options + * including config options related to compilation + * @return a shared pointer on an object implementing NetworkDescription interface + */ + virtual NetworkDescription compile(const std::shared_ptr& model, const Config& config) const = 0; + + /** + * @brief Returns information about supported layers of the network passed + * @param model The model to be queried + * @param config A reference to NPUConfig containing plugin config options + * including config options related to compilation + * @returns SupportedOpsMap structure with information about supported layers + */ + virtual ov::SupportedOpsMap query(const std::shared_ptr& model, const Config& config) const = 0; + + /** + * @brief Parses already compiled network to extract meta information: + * inputs and outputs descriptions + * @param network compiled network represented as a vector of char + * @param config a reference to NPUConfig containing plugin config options + * Note: compilation options will be ignored, + * since the network is already compiled + * @param netName a reference to the string describing network name + * to be used for creating network description + * @return a shared pointer on an object implementing NetworkDescription interface + */ + virtual NetworkMetadata parse(const std::vector& network, const Config& config) const = 0; + + virtual std::vector process_profiling_output(const std::vector& profData, + const std::vector& network, + const Config& config) const = 0; + +protected: + virtual ~ICompiler() = default; +}; + +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/al/include/intel_npu/al/itt.hpp b/src/plugins/intel_npu/src/al/include/intel_npu/al/itt.hpp new file mode 100644 index 00000000000..28b4f1932a5 --- /dev/null +++ b/src/plugins/intel_npu/src/al/include/intel_npu/al/itt.hpp @@ -0,0 +1,20 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/itt.hpp" + +namespace intel_npu { +namespace itt { +namespace domains { + +OV_ITT_DOMAIN(NPUPlugin); +OV_ITT_DOMAIN(LevelZeroBackend); + +} // namespace domains +} // namespace itt +} // namespace intel_npu + +namespace itt = ::intel_npu::itt; diff --git a/src/plugins/intel_npu/src/al/include/intel_npu/al/prefix.hpp b/src/plugins/intel_npu/src/al/include/intel_npu/al/prefix.hpp new file mode 100644 index 00000000000..5bf1744d9b4 --- /dev/null +++ b/src/plugins/intel_npu/src/al/include/intel_npu/al/prefix.hpp @@ -0,0 +1,28 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +namespace intel_npu { + +// +// Prefix for ReadValue and Assign operations in compiler. +// +#define READVALUE_PREFIX std::string("vpux_ie_read_value_") +#define ASSIGN_PREFIX std::string("vpux_ie_assign_") + +inline bool isStateInputName(const std::string& name) { + return !name.compare(0, READVALUE_PREFIX.length(), READVALUE_PREFIX); +} +inline bool isStateOutputName(const std::string& name) { + return !name.compare(0, ASSIGN_PREFIX.length(), ASSIGN_PREFIX); +} + +inline std::string stateOutputToStateInputName(const std::string& name) { + return READVALUE_PREFIX + name.substr(ASSIGN_PREFIX.length()); +} + +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/al/include/intel_npu/al/profiling.hpp b/src/plugins/intel_npu/src/al/include/intel_npu/al/profiling.hpp new file mode 100644 index 00000000000..69b8e80aad6 --- /dev/null +++ b/src/plugins/intel_npu/src/al/include/intel_npu/al/profiling.hpp @@ -0,0 +1,40 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include "openvino/runtime/profiling_info.hpp" + +namespace intel_npu::profiling { + +using LayerStatistics = std::vector; + +template +LayerStatistics convertLayersToIeProfilingInfo(const std::vector& layerInfo) { + LayerStatistics perfCounts; + + perfCounts.reserve(layerInfo.size()); + for (const auto& layer : layerInfo) { + ov::ProfilingInfo& info = perfCounts.emplace_back(); + info.status = ov::ProfilingInfo::Status::EXECUTED; + const auto real_time_ns = std::chrono::nanoseconds(layer.duration_ns); + info.real_time = std::chrono::duration_cast(real_time_ns); + const auto cpu_time_ns = std::chrono::nanoseconds(layer.dma_ns + layer.sw_ns + layer.dpu_ns); + info.cpu_time = std::chrono::duration_cast(cpu_time_ns); + info.node_name = layer.name; + if (layer.sw_ns > 0) { + info.exec_type = "Shave"; + } else if (layer.dpu_ns > 0) { + info.exec_type = "DPU"; + } else { + info.exec_type = "DMA"; + } + info.node_type = layer.layer_type; + } + return perfCounts; +} + +} // namespace intel_npu::profiling diff --git a/src/plugins/intel_npu/src/al/include/npu.hpp b/src/plugins/intel_npu/src/al/include/npu.hpp new file mode 100644 index 00000000000..0bfc660a96c --- /dev/null +++ b/src/plugins/intel_npu/src/al/include/npu.hpp @@ -0,0 +1,74 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include "intel_npu/al/config/config.hpp" +#include "intel_npu/al/icompiled_model.hpp" +#include "intel_npu/al/icompiler.hpp" +#include "openvino/runtime/properties.hpp" +#include "sync_infer_request.hpp" + +namespace intel_npu { + +//------------------------------------------------------------------------------ +class IDevice; + +class IEngineBackend : public std::enable_shared_from_this { +public: + /** @brief Get device, which can be used for inference. Backend responsible for selection. */ + virtual const std::shared_ptr getDevice() const; + /** @brief Search for a specific device by name */ + virtual const std::shared_ptr getDevice(const std::string& specificDeviceName) const; + /** @brief Get device, which is configured/suitable for provided params */ + virtual const std::shared_ptr getDevice(const ov::AnyMap& paramMap) const; + /** @brief Provide a list of names of all devices, with which user can work directly */ + virtual const std::vector getDeviceNames() const; + /** @brief Get name of backend */ + virtual const std::string getName() const = 0; + /** @brief Register backend-specific options */ + virtual void registerOptions(OptionsDesc& options) const; + +protected: + virtual ~IEngineBackend() = default; +}; + +//------------------------------------------------------------------------------ + +class IExecutor { +public: + virtual ~IExecutor() = default; +}; + +//------------------------------------------------------------------------------ + +class IDevice : public std::enable_shared_from_this { +public: + using Uuid = ov::device::UUID; + + virtual std::shared_ptr createExecutor( + const std::shared_ptr& networkDescription, + const Config& config) = 0; + + virtual std::string getName() const = 0; + virtual std::string getFullDeviceName() const = 0; + virtual Uuid getUuid() const; + virtual uint32_t getSubDevId() const; + virtual uint32_t getMaxNumSlices() const; + virtual uint64_t getAllocMemSize() const; + virtual uint64_t getTotalMemSize() const; + virtual uint32_t getDriverVersion() const; + + virtual std::shared_ptr createInferRequest( + const std::shared_ptr& compiledModel, + const std::shared_ptr& executor, + const Config& config) = 0; + +protected: + virtual ~IDevice() = default; +}; + +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/al/include/npu_private_properties.hpp b/src/plugins/intel_npu/src/al/include/npu_private_properties.hpp new file mode 100644 index 00000000000..fd263e1f5f6 --- /dev/null +++ b/src/plugins/intel_npu/src/al/include/npu_private_properties.hpp @@ -0,0 +1,347 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/runtime/intel_npu/properties.hpp" + +namespace ov { +namespace intel_npu { + +namespace Platform { + +constexpr std::string_view AUTO_DETECT = "AUTO_DETECT"; // Auto detection +constexpr std::string_view NPU3700 = "3700"; // NPU30XX +constexpr std::string_view NPU3720 = "3720"; // NPU37XX + +/** + * @brief Converts the given platform value to the standard one. + * @details The same platform value can be defined in multiple ways (e.g. "3720" vs "VPU3720" vs "NPU3720"). The current + * function converts the prefixed variants to the non-prefixed ones in order to enable the comparison between platform + * values. + * + * The values already found in the standard form are returned as they are. + * + * @param platform The value to be converted. + * @return The same platform value given as parameter but converted to the standard form. + */ +inline std::string standardize(const std::string_view platform) { + constexpr std::string_view VPUPrefix = "VPU"; + constexpr std::string_view NPUPrefix = "NPU"; + + if (!platform.compare(0, VPUPrefix.length(), VPUPrefix) || !platform.compare(0, NPUPrefix.length(), NPUPrefix)) { + return std::string(platform).substr(NPUPrefix.length()); + } + + return std::string(platform); +} + +} // namespace Platform + +/** + * @enum ColorFormat + * @brief Extra information about input color format for preprocessing + * @note Configuration API v 2.0 + */ +enum ColorFormat : uint32_t { + RAW = 0u, ///< Plain blob (default), no extra color processing required + RGB, ///< RGB color format + BGR, ///< BGR color format, default in DLDT + RGBX, ///< RGBX color format with X ignored during inference + BGRX, ///< BGRX color format with X ignored during inference +}; + +/** + * @brief Prints a string representation of ov::intel_npu::ColorFormat to a stream + * @param out An output stream to send to + * @param fmt A color format value to print to a stream + * @return A reference to the `out` stream + * @note Configuration API v 2.0 + */ +inline std::ostream& operator<<(std::ostream& out, const ColorFormat& fmt) { + switch (fmt) { + case ColorFormat::RAW: { + out << "RAW"; + } break; + case ColorFormat::RGB: { + out << "RGB"; + } break; + case ColorFormat::BGR: { + out << "BGR"; + } break; + case ColorFormat::RGBX: { + out << "RGBX"; + } break; + case ColorFormat::BGRX: { + out << "BGRX"; + } break; + default: + out << static_cast(fmt); + break; + } + return out; +} + +/** + * @brief [Only for NPU Plugin] + * Type: string, default is MLIR. + * Type of NPU compiler to be used for compilation of a network + * @note Configuration API v 2.0 + */ +enum class CompilerType { MLIR, DRIVER }; + +/** + * @brief Prints a string representation of ov::intel_npu::CompilerType to a stream + * @param out An output stream to send to + * @param fmt A compiler type value to print to a stream + * @return A reference to the `out` stream + * @note Configuration API v 2.0 + */ +inline std::ostream& operator<<(std::ostream& out, const CompilerType& fmt) { + switch (fmt) { + case CompilerType::MLIR: { + out << "MLIR"; + } break; + case CompilerType::DRIVER: { + out << "DRIVER"; + } break; + default: + out << static_cast(fmt); + break; + } + return out; +} + +/** + * @brief [Only for NPU Plugin] + * Type: String. Default is "AUTO". + * This option is added for enabling ELF backend. + * Possible values: "AUTO", "YES", "NO". + */ + +enum class ElfCompilerBackend { + AUTO = 0, + NO = 1, + YES = 2, +}; + +/** + * @brief Prints a string representation of ov::intel_npu::ElfCompilerBackend to a stream + * @param out An output stream to send to + * @param fmt A elf compiler backend value to print to a stream + * @return A reference to the `out` stream + * @note Configuration API v 2.0 + */ +inline std::ostream& operator<<(std::ostream& out, const ElfCompilerBackend& fmt) { + switch (fmt) { + case ElfCompilerBackend::AUTO: { + out << "AUTO"; + } break; + case ElfCompilerBackend::NO: { + out << "NO"; + } break; + case ElfCompilerBackend::YES: { + out << "YES"; + } break; + default: + out << static_cast(fmt); + break; + } + return out; +} + +/** + * @brief [Only for NPU Plugin] + * Type: string, default is MODEL. + * Type of profiling to execute. Can be Model (default) or INFER (based on npu timestamps) + * @note Configuration API v 2.0 + */ +enum class ProfilingType { MODEL, INFER }; + +/** + * @brief Prints a string representation of ov::intel_npu::ProfilingType to a stream + * @param out An output stream to send to + * @param fmt A profiling type value to print to a stream + * @return A reference to the `out` stream + * @note Configuration API v 2.0 + */ +inline std::ostream& operator<<(std::ostream& out, const ProfilingType& fmt) { + switch (fmt) { + case ProfilingType::MODEL: { + out << "MODEL"; + } break; + case ProfilingType::INFER: { + out << "INFER"; + } break; + default: + out << static_cast(fmt); + break; + } + return out; +} + +/** + * @brief Defines the options corresponding to the legacy set of values. + */ +enum class LegacyPriority { + LOW = 0, //!< Low priority + MEDIUM = 1, //!< Medium priority + HIGH = 2 //!< High priority +}; + +inline std::ostream& operator<<(std::ostream& os, const LegacyPriority& priority) { + switch (priority) { + case LegacyPriority::LOW: + return os << "MODEL_PRIORITY_LOW"; + case LegacyPriority::MEDIUM: + return os << "MODEL_PRIORITY_MED"; + case LegacyPriority::HIGH: + return os << "MODEL_PRIORITY_HIGH"; + default: + OPENVINO_THROW("Unsupported model priority value"); + } +} + +inline std::istream& operator>>(std::istream& is, LegacyPriority& priority) { + std::string str; + is >> str; + if (str == "MODEL_PRIORITY_LOW") { + priority = LegacyPriority::LOW; + } else if (str == "MODEL_PRIORITY_MED") { + priority = LegacyPriority::MEDIUM; + } else if (str == "MODEL_PRIORITY_HIGH") { + priority = LegacyPriority::HIGH; + } else { + OPENVINO_THROW("Unsupported model priority: ", str); + } + return is; +} + +/** + * @brief Due to driver compatibility constraints, the set of model priority values corresponding to the OpenVINO legacy + * API is being maintained here. + * @details The OpenVINO API has made changes with regard to the values used to describe model priorities (e.g. + * "MODEL_PRIORITY_MED" -> "MEDIUM"). The NPU plugin can't yet discard this since the newer values may not be + * recognized by older drivers. + */ +static constexpr ov::Property legacy_model_priority{"MODEL_PRIORITY"}; + +/** + * @brief [Only for NPU Plugin] + * Type: Arbitrary string. + * This option allows to specify device. + * The plugin accepts any value given through this option. If the device is not available, either the driver or the + * compiler will throw an exception depending on the flow running at the time. + */ +static constexpr ov::Property platform{"NPU_PLATFORM"}; + +/** + * @brief + * Type: integer, default is -1 + * Device stepping ID. If unset, it will be automatically obtained from driver + */ +static constexpr ov::Property stepping{"NPU_STEPPING"}; + +/** + * @brief [Only for NPU Plugin] + * Type: string, default is DRIVER. + * Selects the type of NPU compiler to be used for compilation of a network. + * 'DRIVER' is the default value. + */ +static constexpr ov::Property compiler_type{"NPU_COMPILER_TYPE"}; + +/** + * @brief + * Selects different compilation pipelines. + */ +static constexpr ov::Property compilation_mode{"NPU_COMPILATION_MODE"}; + +/** + * @brief [Only for NPU compiler] + * Type: std::string, default is empty. + * Sets various parameters supported by the NPU compiler. + * Available values: low-precision=true/low-precision=false + */ +static constexpr ov::Property compilation_mode_params{"NPU_COMPILATION_MODE_PARAMS"}; + +/** + * @brief [Only for NPU Plugin] + * Type: integer, default is None + * Number of DPU groups + */ +static constexpr ov::Property dpu_groups{"NPU_DPU_GROUPS"}; + +/** + * @brief [Only for NPU Compiler] + * Type: integer, default is -1 + * Sets the number of npu tiles that will be used to execute the model. (Replaces NPU_DPU_GROUPS) + */ +static constexpr ov::Property tiles{"NPU_TILES"}; + +/** + * @brief + * Type: integer, default is -1 + * Maximum number of tiles supported by the device. If unset, it will be automatically obtained from driver + */ +static constexpr ov::Property max_tiles{"NPU_MAX_TILES"}; + +/** + * @brief [Only for NPU Plugin] + * Type: integer, default is -1 + * Sets the number of DMA engines that will be used to execute the model. + */ +static constexpr ov::Property dma_engines{"NPU_DMA_ENGINES"}; + +/** + * @brief + * Type: Boolean. Default is "NO". + * Determines which branch we use for dynamic shapes. + * If set to 'YES', we immediately apply the bounds so that we have a static shape for further work. + * If not, we store the related information in TensorAttr and the IE representation looks + * like this: tensor<1x?x3xf32, {bounds = [1, 18, 3], ..}>. + * Possible values: "YES", "NO". + */ +static constexpr ov::Property dynamic_shape_to_static{"NPU_DYNAMIC_SHAPE_TO_STATIC"}; + +/** + * @brief [Only for NPU Plugin] + * Type: string, default is empty. + * MODEL - model layer profiling is done + * INFER - npu inference performance numbers are measured + * Model layers profiling are used if this string is empty + */ +static constexpr ov::Property profiling_type{"NPU_PROFILING_TYPE"}; + +/** + * @brief + * Type: String. Default is "AUTO". + * Sets the format in which the compiled model is stored. + * Possible values: "AUTO", "YES", "NO". + */ +static constexpr ov::Property use_elf_compiler_backend{"NPU_USE_ELF_COMPILER_BACKEND"}; + +/** + * @brief [Only for NPU Plugin] + * Type: integer, default is 1 + * This option allows to omit creating an executor and therefore to omit running an inference when its value is 0 + */ +static constexpr ov::Property create_executor{"NPU_CREATE_EXECUTOR"}; + +/** + * @brief Read-only property to get the name of used backend + */ +static constexpr ov::Property backend_name{"NPU_BACKEND_NAME"}; + +/** + * @brief [Only for NPU compiler] + * Type: std::string, default is empty. + * Config for Backend pipeline + + * Available values: enable-memory-side-cache=true/false + * Available values: enable-partial-workload-management=true/false + */ +static constexpr ov::Property backend_compilation_params{"NPU_BACKEND_COMPILATION_PARAMS"}; + +} // namespace intel_npu +} // namespace ov diff --git a/src/plugins/intel_npu/src/al/include/sync_infer_request.hpp b/src/plugins/intel_npu/src/al/include/sync_infer_request.hpp new file mode 100644 index 00000000000..dd0f2f78809 --- /dev/null +++ b/src/plugins/intel_npu/src/al/include/sync_infer_request.hpp @@ -0,0 +1,186 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "intel_npu/al/icompiled_model.hpp" +#include "intel_npu/al/icompiler.hpp" +#include "openvino/runtime/iinfer_request.hpp" +#include "openvino/runtime/iplugin.hpp" +#include "variable_state.hpp" + +namespace intel_npu { + +/** + * @brief Acts as an interface for the inference request structures implemented by all backends. + * @details The operations common for all backends can be found implemented here, these include tensors' extraction, + * state variables handling and and a few helper functions. + * @note Regarding the code design, the "ov::ISyncInferRequest" would normally be the better option for + * inheritance. However, the interface exposed by that class forces some additional latency in several unfavorable + * scenarios, thus a reimplementation was required. + */ +class SyncInferRequest : public ov::IInferRequest { +public: + explicit SyncInferRequest(const std::shared_ptr& compiledModel); + + /** + * @brief Gets an input/output tensor for inference. + * @note If the tensor with the specified @p port is not found, an exception is thrown. + * @param port Port of the tensor to get. + * @return Tensor for the port @p port. + */ + ov::SoPtr get_tensor(const ov::Output& port) const override; + + /** + * @brief Sets an input/output tensor to infer. + * @param port Port of the input or output tensor. + * @param tensor Reference to a tensor. The element_type and shape of a tensor must match + * the model's input/output element_type and size. + */ + void set_tensor(const ov::Output& port, const ov::SoPtr& tensor) override; + + /** + * @brief Currently there is no support implemented for batches of tensors, thus this call is a simple redirection + * to the "get_tensor" one. + */ + std::vector> get_tensors(const ov::Output& port) const override; + + /** + * @brief Currently there is no support implemented for batches of tensors, thus this call is a simple redirection + * to the "set_tensor" one. + */ + void set_tensors(const ov::Output& port, + const std::vector>& tensors) override; + + /** + * @brief Gets inputs for infer request + * + * @return vector of input ports + */ + const std::vector>& get_inputs() const override; + + /** + * @brief Gets outputs for infer request + * + * @return vector of output ports + */ + const std::vector>& get_outputs() const override; + + /** + * @brief Gets pointer to compiled model (usually synchronous request holds the compiled model) + * + * @return Pointer to the compiled model + */ + const std::shared_ptr& get_compiled_model() const override; + + /** + * @brief Used for executing the inference. + */ + virtual void infer_async() = 0; + + /** + * @brief Used for retrieving the prediction's result. + */ + virtual void get_result() = 0; + + std::vector> query_state() const override; + + /** + * @brief Initializes the tensor values corresponding to the state variables. + * @details The inital values are usually all 0s. + */ + void initialize_states(); + + /** + * @return The state tensors accessible by their names. + */ + std::unordered_map>& get_variable_states() { + return _variableStates; + } + + /** + * @return The names used by the inputs in the order registered inside the model. + */ + std::vector get_input_names() { + return _metadata.inputNames; + } + + /** + * @return The names used by the outputs in the order registered inside the model. + */ + std::vector get_output_names() { + return _metadata.outputNames; + } + + /** + * @return The names used by the state variables in the order registered inside the model. + */ + std::vector get_state_names() { + return _metadata.stateNames; + } + + /** + * @return A map holding references towards all tensors used by the current inference request object. + */ + std::unordered_map>& get_all_tensors() { + return _allTensors; + } + +protected: + /** + * @brief Basic checks for input/output tensor + * + * @param port Input/Output port + * @param tensor Input/Output tensor + */ + void check_tensor(const ov::Output& port, const ov::SoPtr& tensor) const; + + /** + * @brief Check that all tensors are valid. Throws an exception if it's not. + */ + void check_tensors() const override; + + /** + * @brief Checks if the provided precision value is supported by the current backend, should throw an error + * otherwise. + * @param precision The precision value to be checked. + */ + virtual void check_network_precision(const ov::element::Type_t precision) = 0; + + /** + * @brief Allocates a tensor on host and stores the reference inside the "_allTensors" attribute. If a buffer + * address is provided, then the tensor is built upon it and no additional data buffer is allocated. + * @param tensorName The name by which the tensor shall be identified + * @param descriptor Tensor's metadata + * @param isState If true, the tensor shall also be stored inside the state variables map. In this case, adding the + * tensor to this structure would be required in order to correctly answer the state queries. + * @param allocator If provided, the tensor uses the custom allocator instead of using the default one. + */ + void allocate_tensor(std::string tensorName, + const IONodeDescriptor& descriptor, + const bool isState, + const ov::Allocator& allocator = {}); + + // Mutable to return reference to ov::Tensor + mutable std::unordered_map> _allTensors; + // A copy of each tensor is needed to maintain the original L0 memory allocation in case the user provides another + // memory area for the tensor. + std::unordered_map> _copyAllTensors; + + std::unordered_map> _variableStates; + + // This is intel_npu::ICompiledModel pointer, but need to use OV base class because + // ov::IInferRequest::get_compiled_model returns a refernce to shared_ptr! + std::shared_ptr _compiledModel; + + NetworkMetadata _metadata; + + // Stored in order to avoid additional processing when launching inferences + std::vector _inputAndStateInputNames; + std::vector _outputAndStateOutputNames; + + std::unordered_map _nodeNameToLegacyName; +}; + +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/al/include/variable_state.hpp b/src/plugins/intel_npu/src/al/include/variable_state.hpp new file mode 100644 index 00000000000..1a0e574f138 --- /dev/null +++ b/src/plugins/intel_npu/src/al/include/variable_state.hpp @@ -0,0 +1,34 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/runtime/itensor.hpp" +#include "openvino/runtime/ivariable_state.hpp" + +namespace intel_npu { + +class VariableState final : public ov::IVariableState { +public: + explicit VariableState(const std::string& name, const std::shared_ptr& tensor) + : ov::IVariableState(name) { + m_state = tensor; + } + + void set_state(const ov::SoPtr& newState) override { + if (newState->get_byte_size() != m_state->get_byte_size()) { + OPENVINO_THROW("Byte size mismatch"); + } + + std::memcpy(m_state->data(), newState->data(), newState->get_byte_size()); + } + + void reset() override { + std::memset(m_state->data(), 0, m_state->get_byte_size()); + } + + ~VariableState() override = default; +}; + +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/al/src/config/common.cpp b/src/plugins/intel_npu/src/al/src/config/common.cpp new file mode 100644 index 00000000000..c1e0bd2ca27 --- /dev/null +++ b/src/plugins/intel_npu/src/al/src/config/common.cpp @@ -0,0 +1,55 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "intel_npu/al/config/common.hpp" + +using namespace intel_npu; +using namespace ov::intel_npu; + +// +// register +// + +void intel_npu::registerCommonOptions(OptionsDesc& desc) { + desc.add(); + desc.add(); + desc.add(); + desc.add(); + desc.add(); + desc.add(); + desc.add(); + desc.add(); + desc.add(); +} + +// +// PERFORMANCE_HINT +// + +std::string_view ov::hint::stringifyEnum(PerformanceMode val) { + switch (val) { + case PerformanceMode::LATENCY: + return "LATENCY"; + case PerformanceMode::THROUGHPUT: + return "THROUGHPUT"; + case PerformanceMode::CUMULATIVE_THROUGHPUT: + return "CUMULATIVE_THROUGHPUT"; + default: + return ""; + } +} + +ov::hint::PerformanceMode intel_npu::PERFORMANCE_HINT::parse(std::string_view val) { + if (val.empty()) { + return ov::hint::PerformanceMode::LATENCY; + } else if (val == "LATENCY") { + return ov::hint::PerformanceMode::LATENCY; + } else if (val == "THROUGHPUT") { + return ov::hint::PerformanceMode::THROUGHPUT; + } else if (val == "CUMULATIVE_THROUGHPUT") { + return ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT; + } + + OPENVINO_THROW("Value '", val, "' is not a valid PERFORMANCE_HINT option"); +} diff --git a/src/plugins/intel_npu/src/al/src/config/compiler.cpp b/src/plugins/intel_npu/src/al/src/config/compiler.cpp new file mode 100644 index 00000000000..aaa98da362f --- /dev/null +++ b/src/plugins/intel_npu/src/al/src/config/compiler.cpp @@ -0,0 +1,121 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "intel_npu/al/config/compiler.hpp" + +using namespace intel_npu; +using namespace ov::intel_npu; + +// +// register +// + +void intel_npu::registerCompilerOptions(OptionsDesc& desc) { + desc.add(); + desc.add(); + desc.add(); + desc.add(); + desc.add(); + desc.add(); + desc.add(); + desc.add(); + desc.add(); + desc.add(); + desc.add(); + desc.add(); +} + +// +// COMPILER_TYPE +// + +std::string_view ov::intel_npu::stringifyEnum(ov::intel_npu::CompilerType val) { + switch (val) { + case ov::intel_npu::CompilerType::MLIR: + return "MLIR"; + case ov::intel_npu::CompilerType::DRIVER: + return "DRIVER"; + default: + return ""; + } +} + +std::string_view intel_npu::COMPILER_TYPE::envVar() { +#ifdef NPU_PLUGIN_DEVELOPER_BUILD + return "IE_NPU_COMPILER_TYPE"; +#else + return ""; +#endif +} + +ov::intel_npu::CompilerType intel_npu::COMPILER_TYPE::defaultValue() { + return ov::intel_npu::CompilerType::DRIVER; +} + +ov::intel_npu::CompilerType intel_npu::COMPILER_TYPE::parse(std::string_view val) { + if (val == stringifyEnum(ov::intel_npu::CompilerType::MLIR)) { + return ov::intel_npu::CompilerType::MLIR; + } else if (val == stringifyEnum(ov::intel_npu::CompilerType::DRIVER)) { + return ov::intel_npu::CompilerType::DRIVER; + } + + OPENVINO_THROW("Value '", val, "' is not a valid COMPILER_TYPE option"); +} + +std::string intel_npu::COMPILER_TYPE::toString(const ov::intel_npu::CompilerType& val) { + std::stringstream strStream; + if (val == ov::intel_npu::CompilerType::MLIR) { + strStream << "MLIR"; + } else if (val == ov::intel_npu::CompilerType::DRIVER) { + strStream << "DRIVER"; + } else { + OPENVINO_THROW("No valid string for current LOG_LEVEL option"); + } + + return strStream.str(); +} + +// +// USE_ELF_COMPILER_BACKEND +// + +std::string_view ov::intel_npu::stringifyEnum(ov::intel_npu::ElfCompilerBackend val) { + switch (val) { + case ov::intel_npu::ElfCompilerBackend::AUTO: + return "AUTO"; + case ov::intel_npu::ElfCompilerBackend::NO: + return "NO"; + case ov::intel_npu::ElfCompilerBackend::YES: + return "YES"; + default: + return ""; + } +} + +ov::intel_npu::ElfCompilerBackend intel_npu::USE_ELF_COMPILER_BACKEND::parse(std::string_view val) { + if (val == stringifyEnum(ov::intel_npu::ElfCompilerBackend::AUTO)) { + return ov::intel_npu::ElfCompilerBackend::AUTO; + } else if (val == stringifyEnum(ov::intel_npu::ElfCompilerBackend::NO)) { + return ov::intel_npu::ElfCompilerBackend::NO; + } else if (val == stringifyEnum(ov::intel_npu::ElfCompilerBackend::YES)) { + return ov::intel_npu::ElfCompilerBackend::YES; + } + + OPENVINO_THROW("Value '", val, "' is not a valid USE_ELF_COMPILER_BACKEND option"); +} + +std::string intel_npu::USE_ELF_COMPILER_BACKEND::toString(const ov::intel_npu::ElfCompilerBackend& val) { + std::stringstream strStream; + if (val == ov::intel_npu::ElfCompilerBackend::AUTO) { + strStream << "AUTO"; + } else if (val == ov::intel_npu::ElfCompilerBackend::NO) { + strStream << "NO"; + } else if (val == ov::intel_npu::ElfCompilerBackend::YES) { + strStream << "YES"; + } else { + OPENVINO_THROW("No valid string for current USE_ELF_COMPILER_BACKEND option"); + } + + return strStream.str(); +} diff --git a/src/plugins/intel_npu/src/al/src/config/config.cpp b/src/plugins/intel_npu/src/al/src/config/config.cpp new file mode 100644 index 00000000000..1f104f5626a --- /dev/null +++ b/src/plugins/intel_npu/src/al/src/config/config.cpp @@ -0,0 +1,241 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "intel_npu/al/config/config.hpp" + +namespace intel_npu { + +// Splits the `str` string onto separate elements using `delim` as delimiter and +// call `callback` for each element. +void splitAndApply(const std::string& str, char delim, std::function callback) { + const auto begin = str.begin(); + const auto end = str.end(); + + auto curBegin = begin; + auto curEnd = begin; + while (curEnd != end) { + while (curEnd != end && *curEnd != delim) { + ++curEnd; + } + + callback(std::string_view(&(*curBegin), static_cast(curEnd - curBegin))); + + if (curEnd != end) { + ++curEnd; + curBegin = curEnd; + } + } +} + +// +// OptionParser +// + +bool OptionParser::parse(std::string_view val) { + if (val == "YES") { + return true; + } else if (val == "NO") { + return false; + } + + OPENVINO_THROW("Value '", val.data(), "' is not a valid BOOL option"); +} + +int32_t OptionParser::parse(std::string_view val) { + try { + return std::stol(val.data()); + } catch (...) { + OPENVINO_THROW("Value '%s' is not a valid INT32 option", val.data()); + } +} + +int64_t OptionParser::parse(std::string_view val) { + try { + return std::stoll(val.data()); + } catch (...) { + OPENVINO_THROW("Value '", val.data(), "' is not a valid INT64 option"); + } +} + +uint64_t OptionParser::parse(std::string_view val) { + try { + return std::stoull(val.data()); + } catch (...) { + OPENVINO_THROW("Value '", val.data(), "' is not a valid UINT64 option"); + } +} + +double OptionParser::parse(std::string_view val) { + try { + return std::stod(val.data()); + } catch (...) { + OPENVINO_THROW("Value '", val.data(), "' is not a valid FP64 option"); + } +} + +ov::log::Level OptionParser::parse(std::string_view val) { + std::string strVal(val); + std::istringstream is(strVal); + ov::log::Level level; + is >> level; + return level; +} + +// +// OptionPrinter +// + +std::string OptionPrinter::toString(bool val) { + return val ? "YES" : "NO"; +} + +std::string OptionPrinter::toString(ov::log::Level val) { + std::ostringstream os; + os << val; + return os.str(); +} + +// +// OptionMode +// + +std::string_view stringifyEnum(OptionMode val) { + switch (val) { + case OptionMode::Both: + return "Both"; + case OptionMode::CompileTime: + return "CompileTime"; + case OptionMode::RunTime: + return "RunTime"; + default: + return ""; + } +} + +// +// OptionValue +// + +details::OptionValue::~OptionValue() = default; + +// +// OptionsDesc +// + +details::OptionConcept OptionsDesc::get(std::string_view key, OptionMode mode) const { + auto log = Logger::global().clone("OptionsDesc"); + + std::string searchKey{key}; + const auto itDeprecated = _deprecated.find(std::string(key)); + if (itDeprecated != _deprecated.end()) { + searchKey = itDeprecated->second; + log.warning("Deprecated option '%s' was used, '%s' should be used instead", key.data(), searchKey.c_str()); + } + + const auto itMain = _impl.find(searchKey); + OPENVINO_ASSERT(itMain != _impl.end(), + "[ NOT_FOUND ] Option '", + key.data(), + "' is not supported for current configuration"); + + const auto& desc = itMain->second; + + if (mode == OptionMode::RunTime) { + if (desc.mode() == OptionMode::CompileTime) { + log.warning("%s option '%s' was used in %s mode", + stringifyEnum(desc.mode()).data(), + key.data(), + stringifyEnum(mode).data()); + } + } + + return desc; +} + +std::vector OptionsDesc::getSupported(bool includePrivate) const { + std::vector res; + res.reserve(_impl.size()); + + for (const auto& p : _impl) { + if (p.second.isPublic() || includePrivate) { + res.push_back(p.first); + } + } + + return res; +} + +void OptionsDesc::walk(std::function cb) const { + for (const auto& [_, opt] : _impl) { + cb(opt); + } +} + +// +// Config +// + +Config::Config(const std::shared_ptr& desc) : _desc(desc) { + OPENVINO_ASSERT(_desc != nullptr, "Got NULL OptionsDesc"); +} + +void Config::parseEnvVars() { + auto log = Logger::global().clone("Config"); + + _desc->walk([&](const details::OptionConcept& opt) { + if (!opt.envVar().empty()) { + if (const auto envVar = std::getenv(opt.envVar().data())) { + log.trace("Update option '%s' to value '%s' parsed from environment variable '%s'", + opt.key().data(), + envVar, + opt.envVar().data()); + + _impl[opt.key().data()] = opt.validateAndParse(envVar); + } + } + }); +} + +void Config::update(const ConfigMap& options, OptionMode mode) { + auto log = Logger::global().clone("Config"); + + for (const auto& p : options) { + log.trace("Update option '%s' to value '%s'", p.first.c_str(), p.second.c_str()); + + const auto opt = _desc->get(p.first, mode); + _impl[opt.key().data()] = opt.validateAndParse(p.second); + } +} + +std::string Config::toString() const { + std::stringstream resultStream; + for (auto it = _impl.cbegin(); it != _impl.cend(); ++it) { + const auto key = it->first; + + resultStream << key << "=\"" << it->second->toString() << "\""; + if (std::next(it) != _impl.end()) { + resultStream << " "; + } + } + + return resultStream.str(); +} + +// +// envVarStrToBool +// + +bool envVarStrToBool(const char* varName, const char* varValue) { + try { + const auto intVal = std::stoi(varValue); + if (intVal != 0 && intVal != 1) { + throw std::invalid_argument("Only 0 and 1 values are supported"); + } + return (intVal != 0); + } catch (const std::exception& e) { + OPENVINO_THROW(std::string("Environment variable ") + varName + " has wrong value : " + e.what()); + } +} + +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/al/src/config/runtime.cpp b/src/plugins/intel_npu/src/al/src/config/runtime.cpp new file mode 100644 index 00000000000..bf731d96dd5 --- /dev/null +++ b/src/plugins/intel_npu/src/al/src/config/runtime.cpp @@ -0,0 +1,130 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "intel_npu/al/config/runtime.hpp" + +#include "intel_npu/al/config/common.hpp" + +using namespace intel_npu; +using namespace ov::intel_npu; + +// +// register +// + +void intel_npu::registerRunTimeOptions(OptionsDesc& desc) { + desc.add(); + desc.add(); + desc.add(); + desc.add(); + desc.add(); + desc.add(); +} + +// Heuristically obtained number. Varies depending on the values of PLATFORM and PERFORMANCE_HINT +// Note: this is the value provided by the plugin, application should query and consider it, but may supply its own +// preference for number of parallel requests via dedicated configuration +int64_t intel_npu::getOptimalNumberOfInferRequestsInParallel(const Config& config) { + const std::string platform = ov::intel_npu::Platform::standardize(config.get()); + + if (platform == ov::intel_npu::Platform::NPU3720) { + if (config.get() == ov::hint::PerformanceMode::THROUGHPUT) { + return 4; + } else { + return 1; + } + } else { + if (config.get() == ov::hint::PerformanceMode::THROUGHPUT) { + return 8; + } else { + return 1; + } + } +} + +// +// PROFILING_TYPE +// + +std::string_view ov::intel_npu::stringifyEnum(ov::intel_npu::ProfilingType val) { + switch (val) { + case ov::intel_npu::ProfilingType::MODEL: + return "MODEL"; + case ov::intel_npu::ProfilingType::INFER: + return "INFER"; + default: + return ""; + } +} + +ov::intel_npu::ProfilingType intel_npu::PROFILING_TYPE::parse(std::string_view val) { + const auto extractProfilingString = [](ov::intel_npu::ProfilingType prof) -> std::string { + return profiling_type(prof).second.as(); + }; + + if (val == extractProfilingString(ov::intel_npu::ProfilingType::MODEL)) { + return ov::intel_npu::ProfilingType::MODEL; + } else if (val == extractProfilingString(ov::intel_npu::ProfilingType::INFER)) { + return ov::intel_npu::ProfilingType::INFER; + } + + OPENVINO_THROW("Value '", val, "' is not a valid PROFILING_TYPE option"); +} + +std::string intel_npu::PROFILING_TYPE::toString(const ov::intel_npu::ProfilingType& val) { + std::stringstream strStream; + if (val == ov::intel_npu::ProfilingType::MODEL) { + strStream << "MODEL"; + } else if (val == ov::intel_npu::ProfilingType::INFER) { + strStream << "INFER"; + } else { + OPENVINO_THROW("No valid string for current PROFILING_TYPE option"); + } + + return strStream.str(); +} + +// +// MODEL_PRIORITY +// + +ov::hint::Priority intel_npu::MODEL_PRIORITY::parse(std::string_view val) { + std::istringstream stringStream = std::istringstream(std::string(val)); + ov::hint::Priority priority; + + stringStream >> priority; + + return priority; +} + +std::string intel_npu::MODEL_PRIORITY::toString(const ov::hint::Priority& val) { + std::ostringstream stringStream; + + stringStream << val; + + return stringStream.str(); +} + +// +// NUM_STREAMS +// + +const ov::streams::Num intel_npu::NUM_STREAMS::defVal = ov::streams::Num(1); + +ov::streams::Num intel_npu::NUM_STREAMS::parse(std::string_view val) { + std::istringstream stringStream = std::istringstream(std::string(val)); + ov::streams::Num numberOfStreams; + + stringStream >> numberOfStreams; + + return numberOfStreams; +} + +std::string intel_npu::NUM_STREAMS::toString(const ov::streams::Num& val) { + std::ostringstream stringStream; + + stringStream << val; + + return stringStream.str(); +} diff --git a/src/plugins/intel_npu/src/al/src/device_helpers.cpp b/src/plugins/intel_npu/src/al/src/device_helpers.cpp new file mode 100644 index 00000000000..f06c74971bb --- /dev/null +++ b/src/plugins/intel_npu/src/al/src/device_helpers.cpp @@ -0,0 +1,33 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "device_helpers.hpp" + +#include "openvino/core/except.hpp" + +bool utils::isNPUDevice(const uint32_t deviceId) { + // bits 26-24 define interface type + // 000 - IPC + // 001 - PCIe + // 010 - USB + // 011 - ethernet + constexpr uint32_t INTERFACE_TYPE_SELECTOR = 0x7000000; + uint32_t interfaceType = (deviceId & INTERFACE_TYPE_SELECTOR); + return (interfaceType == 0); +} + +uint32_t utils::getSliceIdBySwDeviceId(const uint32_t swDevId) { + // bits 3-1 define slice ID + // right shift to omit bit 0, thus slice id is stored in bits 2-0 + // apply b111 mask to discard anything but slice ID + uint32_t sliceId = (swDevId >> 1) & 0x7; + return sliceId; +} + +std::string utils::getPlatformByDeviceName(const std::string& deviceName) { + const auto platformPos = deviceName.rfind('.'); + const auto platformName = (platformPos == std::string::npos) ? deviceName : deviceName.substr(0, platformPos); + + return platformName; +} diff --git a/src/plugins/intel_npu/src/al/src/npu.cpp b/src/plugins/intel_npu/src/al/src/npu.cpp new file mode 100644 index 00000000000..f88c0427f7b --- /dev/null +++ b/src/plugins/intel_npu/src/al/src/npu.cpp @@ -0,0 +1,51 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "npu.hpp" + +#include "intel_npu/al/itt.hpp" +#include "openvino/util/shared_object.hpp" + +namespace intel_npu { + +const std::shared_ptr IEngineBackend::getDevice() const { + OPENVINO_THROW("Default getDevice() not implemented"); +} +const std::shared_ptr IEngineBackend::getDevice(const std::string&) const { + OPENVINO_THROW("Specific device search not implemented"); +} +const std::shared_ptr IEngineBackend::getDevice(const ov::AnyMap&) const { + OPENVINO_THROW("Get device based on params not implemented"); +} +const std::vector IEngineBackend::getDeviceNames() const { + OPENVINO_THROW("Get all device names not implemented"); +} + +void IEngineBackend::registerOptions(OptionsDesc&) const {} + +IDevice::Uuid IDevice::getUuid() const { + OPENVINO_THROW("Get UUID not supported"); +} + +uint32_t IDevice::getSubDevId() const { + OPENVINO_THROW("Get SubDevId is not supported"); +} + +uint32_t IDevice::getMaxNumSlices() const { + OPENVINO_THROW("Get MaxNumSlices is not supported"); +} + +uint64_t IDevice::getAllocMemSize() const { + OPENVINO_THROW("Get AllocMemSize is not supported"); +} + +uint64_t IDevice::getTotalMemSize() const { + OPENVINO_THROW("Get TotalMemSize is not supported"); +} + +uint32_t IDevice::getDriverVersion() const { + OPENVINO_THROW("Get NPU driver version is not supported with this backend"); +} + +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/al/src/sync_infer_request.cpp b/src/plugins/intel_npu/src/al/src/sync_infer_request.cpp new file mode 100644 index 00000000000..4a02958d677 --- /dev/null +++ b/src/plugins/intel_npu/src/al/src/sync_infer_request.cpp @@ -0,0 +1,186 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "sync_infer_request.hpp" + +#include "intel_npu/al/prefix.hpp" +#include "openvino/runtime/make_tensor.hpp" +#include "openvino/runtime/plugin_itt.hpp" +#include "transformations/utils/utils.hpp" + +namespace intel_npu { + +SyncInferRequest::SyncInferRequest(const std::shared_ptr& compiledModel) + : _compiledModel(compiledModel), + _metadata(compiledModel->get_network_metadata()) { + OPENVINO_ASSERT(_compiledModel); + + const std::vector>& inputs = get_inputs(); + const std::vector>& outputs = get_outputs(); + + if (inputs.empty()) { + OPENVINO_THROW("Inference request creation: no input found for network " + _metadata.name); + } + if (outputs.empty()) { + OPENVINO_THROW("Inference request creation: no output found for network " + _metadata.name); + } + + // Map the node names to the legacy ones used by the I/O tensors in order to allow an easier access to the tensors' + // contents + for (const auto& [legacyName, parameterDescriptor] : _metadata.parameters) { + _nodeNameToLegacyName[parameterDescriptor.currentNodeName] = legacyName; + } + for (const auto& [legacyName, resultDescriptor] : _metadata.results) { + _nodeNameToLegacyName[resultDescriptor.currentNodeName] = legacyName; + } + + _inputAndStateInputNames = _metadata.inputNames; + _outputAndStateOutputNames = _metadata.outputNames; + + for (const std::string& stateName : _metadata.stateNames) { + // State variables shall be identified by specific prefixes in order to avoid a potential tensor name collision + _inputAndStateInputNames.push_back(READVALUE_PREFIX + stateName); + _outputAndStateOutputNames.push_back(ASSIGN_PREFIX + stateName); + } +} + +const std::vector>& SyncInferRequest::get_inputs() const { + return _compiledModel->inputs(); +} + +const std::vector>& SyncInferRequest::get_outputs() const { + return _compiledModel->outputs(); +} + +const std::shared_ptr& SyncInferRequest::get_compiled_model() const { + return _compiledModel; +} + +void SyncInferRequest::initialize_states() { + for (const std::string& stateName : _metadata.stateNames) { + _variableStates.at(stateName)->reset(); + } +} + +std::vector> SyncInferRequest::query_state() const { + std::vector> queryResult; + + for (const std::string& stateName : _metadata.stateNames) { + queryResult.push_back(_variableStates.at(stateName)); + } + + return queryResult; +} + +ov::SoPtr SyncInferRequest::get_tensor(const ov::Output& port) const { + const auto& nodeNameMatch = _nodeNameToLegacyName.find(port.get_node()->get_friendly_name()); + OPENVINO_ASSERT(nodeNameMatch != _nodeNameToLegacyName.end(), "Cannot find tensor for port ", port); + + return _allTensors.at(nodeNameMatch->second); +} + +void SyncInferRequest::set_tensor(const ov::Output& port, const ov::SoPtr& tensor) { + OV_ITT_SCOPED_TASK(ov::itt::domains::Plugin, "set_tensor"); + try { + check_tensor(port, tensor); + } catch (const ov::Exception& ex) { + OPENVINO_THROW("Failed to set tensor. ", ex.what()); + } + + const std::string& legacyName = _nodeNameToLegacyName.at(port.get_node()->get_friendly_name()); + _allTensors[legacyName] = tensor._ptr; +} + +std::vector> SyncInferRequest::get_tensors(const ov::Output& /*port*/) const { + OV_ITT_SCOPED_TASK(ov::itt::domains::Plugin, "get_tensors"); + // Using batches of tensors is currently not supported by the NPU plugin. In this scenario, the OpenVINO API demands + // returning an empty vector. + return {}; +} + +void SyncInferRequest::set_tensors(const ov::Output& port, + const std::vector>& tensors) { + OV_ITT_SCOPED_TASK(ov::itt::domains::Plugin, "set_tensors"); + if (tensors.size() == 1) { + set_tensor(port, tensors[0]); + return; + } + + OPENVINO_THROW_NOT_IMPLEMENTED("set_input_tensors/set_tensors are not supported by this plugin"); +} + +void SyncInferRequest::check_tensor(const ov::Output& port, + const ov::SoPtr& tensor) const { + if (tensor == nullptr) + OPENVINO_THROW("The tensor is not initialized!"); + + bool is_input = ov::op::util::is_parameter(port.get_node()); + std::string tensor_type = is_input ? "input" : "output"; + + OPENVINO_ASSERT(port.get_element_type() == tensor->get_element_type(), + "The tensor element type is not corresponding with output element type (", + tensor->get_element_type(), + " != ", + port.get_element_type()); + bool is_dynamic = port.get_partial_shape().is_dynamic(); + OPENVINO_ASSERT(is_dynamic || port.get_shape() == tensor->get_shape(), + "The ", + tensor_type, + " tensor size is not equal to the model ", + tensor_type, + " type: got ", + tensor->get_shape(), + " expecting ", + port.get_shape(), + "."); + OPENVINO_ASSERT( + std::dynamic_pointer_cast(tensor._ptr) || tensor->data() != nullptr || is_dynamic, + "Tensor data equal nullptr!"); +} + +void SyncInferRequest::check_tensors() const { + const auto& inputs = _compiledModel->inputs(); + for (size_t i = 0; i < inputs.size(); i++) { + const std::string& legacyName = _nodeNameToLegacyName.at(inputs[i].get_node()->get_friendly_name()); + check_tensor(inputs[i], _allTensors.at(legacyName)); + } + + const auto& outputs = _compiledModel->outputs(); + for (size_t i = 0; i < outputs.size(); i++) { + const std::string& legacyName = _nodeNameToLegacyName.at(outputs[i].get_node()->get_friendly_name()); + check_tensor(outputs[i], _allTensors.at(legacyName)); + } +} + +void SyncInferRequest::allocate_tensor(std::string tensorName, + const IONodeDescriptor& descriptor, + const bool isState, + const ov::Allocator& allocator) { + std::shared_ptr tensor; + + check_network_precision(descriptor.precision); + + if (allocator) { + tensor = ov::make_tensor(descriptor.precision, descriptor.transposedShape.get_shape(), allocator); + } else { + tensor = ov::make_tensor(descriptor.precision, descriptor.transposedShape.get_shape()); + } + + if (!isState) { + _copyAllTensors[tensorName] = std::move(tensor); + _allTensors[tensorName] = _copyAllTensors[tensorName]; + } else { + _variableStates[tensorName] = std::make_shared(tensorName, tensor); + + // State variables shall be identified by specific prefixes in order to avoid a potential tensor name collision. + // Additionally, only one buffer is required in the whole flow, acting as an input before running the inference + // and as an output after performing it. Thus both the "state input" and "state output" entries shall point to + // the same buffer. + _copyAllTensors[READVALUE_PREFIX + tensorName] = std::move(tensor); + _copyAllTensors[ASSIGN_PREFIX + tensorName] = _copyAllTensors[READVALUE_PREFIX + tensorName]; + _allTensors[READVALUE_PREFIX + tensorName] = _copyAllTensors[READVALUE_PREFIX + tensorName]; + _allTensors[ASSIGN_PREFIX + tensorName] = _copyAllTensors[READVALUE_PREFIX + tensorName]; + } +} +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/backend/CMakeLists.txt b/src/plugins/intel_npu/src/backend/CMakeLists.txt new file mode 100644 index 00000000000..375b15bac36 --- /dev/null +++ b/src/plugins/intel_npu/src/backend/CMakeLists.txt @@ -0,0 +1,52 @@ +# Copyright (C) 2018-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +set(TARGET_NAME "openvino_npu_level_zero_backend") + +file(GLOB_RECURSE SOURCES *.cpp *.hpp *.h) +source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR} FILES ${SOURCES}) + +add_library(${TARGET_NAME} STATIC ${SOURCES}) + +set_target_properties(${TARGET_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO}) +ov_add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME}) + +target_compile_definitions(${TARGET_NAME} + PRIVATE + IMPLEMENT_OPENVINO_RUNTIME_PLUGIN) + +target_include_directories(${TARGET_NAME} + PUBLIC + $ +) + +target_link_libraries(${TARGET_NAME} + PRIVATE + openvino::npu_al + openvino_npu_zero_result_parser + ze_loader +) + +# +# targets install +# +ov_install_static_lib(${TARGET_NAME} ${NPU_INTERNAL_COMPONENT}) + +if(TARGET ze_loader) + if(NOT BUILD_SHARED_LIBS) + # Support link of static runtime in case system does not have ze_loader + install(TARGETS ze_loader EXPORT OpenVINOTargets + RUNTIME DESTINATION ${OV_CPACK_RUNTIMEDIR} COMPONENT ${NPU_PLUGIN_COMPONENT} + ARCHIVE DESTINATION ${OV_CPACK_ARCHIVEDIR} COMPONENT ${NPU_PLUGIN_COMPONENT} + LIBRARY DESTINATION ${OV_CPACK_LIBRARYDIR} COMPONENT ${NPU_PLUGIN_COMPONENT}) + + # export to local tree to build against static build tree + export(TARGETS ze_loader NAMESPACE openvino:: + APPEND FILE "${CMAKE_BINARY_DIR}/OpenVINOTargets.cmake") + endif() + + # Support tests to run with ze_loader + install(TARGETS ze_loader + LIBRARY DESTINATION tests COMPONENT tests EXCLUDE_FROM_ALL) +endif() diff --git a/src/plugins/intel_npu/src/backend/include/zero_backend.hpp b/src/plugins/intel_npu/src/backend/include/zero_backend.hpp new file mode 100644 index 00000000000..fd3d03c5079 --- /dev/null +++ b/src/plugins/intel_npu/src/backend/include/zero_backend.hpp @@ -0,0 +1,31 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +#include "npu.hpp" +#include "zero_init.hpp" + +namespace intel_npu { +class ZeroEngineBackend final : public IEngineBackend { +public: + ZeroEngineBackend(const Config& config); + virtual ~ZeroEngineBackend(); + const std::shared_ptr getDevice() const override; + const std::shared_ptr getDevice(const std::string&) const override; + const std::string getName() const override { + return "LEVEL0"; + } + const std::vector getDeviceNames() const override; + +private: + std::shared_ptr _instance; + + std::map> _devices{}; +}; + +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/backend/include/zero_device.hpp b/src/plugins/intel_npu/src/backend/include/zero_device.hpp new file mode 100644 index 00000000000..3ca7f3316de --- /dev/null +++ b/src/plugins/intel_npu/src/backend/include/zero_device.hpp @@ -0,0 +1,53 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +#include "intel_npu/al/icompiled_model.hpp" +#include "intel_npu/utils/logger/logger.hpp" +#include "npu.hpp" +#include "zero_init.hpp" +#include "zero_types.hpp" +#include "zero_utils.hpp" +namespace intel_npu { + +class ZeroDevice : public IDevice { +public: + ZeroDevice(const std::shared_ptr& initStructs); + + std::shared_ptr createExecutor(const std::shared_ptr& networkDescription, + const Config& config) override; + + std::string getName() const override; + std::string getFullDeviceName() const override; + Uuid getUuid() const override; + uint32_t getSubDevId() const override; + uint32_t getMaxNumSlices() const override; + uint64_t getAllocMemSize() const override; + uint64_t getTotalMemSize() const override; + uint32_t getDriverVersion() const override; + + std::shared_ptr createInferRequest(const std::shared_ptr& compiledModel, + const std::shared_ptr& executor, + const Config& config) override; + + ZeroDevice& operator=(const ZeroDevice&) = delete; + ZeroDevice(const ZeroDevice&) = delete; + +private: + const std::shared_ptr _initStructs; + + ze_graph_dditable_ext_curr_t* _graph_ddi_table_ext = nullptr; + + ze_device_properties_t device_properties = {}; + ze_driver_properties_t driver_properties = {}; + + uint32_t _group_ordinal; + + Logger log; +}; +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/backend/include/zero_executor.hpp b/src/plugins/intel_npu/src/backend/include/zero_executor.hpp new file mode 100644 index 00000000000..7ef8d36abcd --- /dev/null +++ b/src/plugins/intel_npu/src/backend/include/zero_executor.hpp @@ -0,0 +1,76 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +#include "intel_npu/utils/logger/logger.hpp" +#include "npu.hpp" +#include "zero_init.hpp" +#include "zero_wrappers.hpp" + +namespace intel_npu { + +class ZeroExecutor final : public IExecutor { +public: + ZeroExecutor(const std::shared_ptr& initStructs, + const std::shared_ptr& networkDescription, + const Config& config, + const uint32_t& group_ordinal); + + ZeroExecutor(const ZeroExecutor&) = delete; + ZeroExecutor& operator=(const ZeroExecutor&) = delete; + + ~ZeroExecutor() override; + + struct ArgumentDescriptor { + ze_graph_argument_properties_t info; + uint32_t idx; + }; + + void setArgumentValue(uint32_t argi_, const void* argv_) const; + inline ze_graph_handle_t graph() const { + return _graph; + } + inline std::shared_ptr getInitStructs() const { + return _initStructs; + } + inline const std::shared_ptr& getNetworkDesc() const { + return _networkDesc; + } + inline const std::array, stage::COUNT>& getCommandQueue() const { + return _command_queues; + } + inline const uint32_t& get_group_ordinal() const { + return _group_ordinal; + } + inline const std::unordered_map& inputs_desc_map() const { + return _inputs_desc_map; + } + inline const std::unordered_map& outputs_desc_map() const { + return _outputs_desc_map; + } + +private: + const Config _config; + Logger _logger; + + const std::shared_ptr _initStructs; + std::shared_ptr _networkDesc; + + ze_graph_dditable_ext_curr_t* _graph_ddi_table_ext = nullptr; + + const uint32_t _group_ordinal; + + ze_graph_handle_t _graph = nullptr; + ze_graph_properties_t _props{}; + std::unordered_map _inputs_desc_map; + std::unordered_map _outputs_desc_map; + + std::array, stage::COUNT> _command_queues; +}; + +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/backend/include/zero_infer_request.hpp b/src/plugins/intel_npu/src/backend/include/zero_infer_request.hpp new file mode 100644 index 00000000000..5295c2e968a --- /dev/null +++ b/src/plugins/intel_npu/src/backend/include/zero_infer_request.hpp @@ -0,0 +1,51 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +#include + +#include "intel_npu/utils/logger/logger.hpp" +#include "npu.hpp" +#include "zero_executor.hpp" +#include "zero_pipeline.hpp" +#include "zero_profiling.hpp" +#include "zero_utils.hpp" +#include "zero_wrappers.hpp" + +namespace intel_npu { + +class ZeroInferRequest final : public SyncInferRequest { +public: + explicit ZeroInferRequest(const std::shared_ptr& backendPtr, + const std::shared_ptr& compiledModel, + const std::shared_ptr& executor, + const Config& config); + + void infer() override; + void infer_async() override; + + void get_result() override; + +private: + std::vector get_profiling_info() const override; + std::vector get_raw_profiling_data() const; + + void check_network_precision(const ov::element::Type_t precision) override; + + const std::shared_ptr _executorPtr; + const ZeroExecutor* _executor; + const Config _config; + Logger _logger; + + zeroProfiling::ProfilingPool _profiling_pool; + zeroProfiling::ProfilingQuery _profiling_query; + std::shared_ptr _npu_profiling; + std::unique_ptr _pipeline; +}; + +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/backend/include/zero_init.hpp b/src/plugins/intel_npu/src/backend/include/zero_init.hpp new file mode 100644 index 00000000000..2c7ad554674 --- /dev/null +++ b/src/plugins/intel_npu/src/backend/include/zero_init.hpp @@ -0,0 +1,57 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +#include + +#include "intel_npu/utils/logger/logger.hpp" +#include "ze_intel_vpu_uuid.h" +#include "zero_types.hpp" + +namespace intel_npu { +/** + * Holder for the level zero structures which must be initialized via call to the driver once zero backend is loaded, + * and de-initialized after their last use is over. + */ +class ZeroInitStructsHolder final { +public: + ZeroInitStructsHolder(); + + ZeroInitStructsHolder(const ZeroInitStructsHolder&) = delete; + ZeroInitStructsHolder& operator=(const ZeroInitStructsHolder&) = delete; + + ~ZeroInitStructsHolder(); + + inline ze_driver_handle_t getDriver() const { + return driver_handle; + } + inline ze_device_handle_t getDevice() const { + return device_handle; + } + inline ze_context_handle_t getContext() const { + return context; + } + inline ze_graph_dditable_ext_curr_t* getGraphDdiTable() const { + return graph_dditable_ext_decorator.get(); + } + inline ze_graph_profiling_dditable_ext_t* getProfilingDdiTable() const { + return _graph_profiling_ddi_table_ext; + } + +private: + static const ze_driver_uuid_t uuid; + Logger log; + + ze_driver_handle_t driver_handle = nullptr; + ze_device_handle_t device_handle = nullptr; + ze_context_handle_t context = nullptr; + std::unique_ptr graph_dditable_ext_decorator; + ze_graph_profiling_dditable_ext_t* _graph_profiling_ddi_table_ext = nullptr; +}; + +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/backend/include/zero_memory.hpp b/src/plugins/intel_npu/src/backend/include/zero_memory.hpp new file mode 100644 index 00000000000..3633ca1a90e --- /dev/null +++ b/src/plugins/intel_npu/src/backend/include/zero_memory.hpp @@ -0,0 +1,121 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +#include +#include + +#include "intel_npu/utils/logger/logger.hpp" +#include "zero_init.hpp" + +namespace { + +constexpr std::size_t STANDARD_PAGE_SIZE = 4096; + +} // namespace + +namespace intel_npu { +namespace zeroMemory { +struct DeviceMem { + DeviceMem() = delete; + DeviceMem(const ze_device_handle_t device_handle, const ze_context_handle_t context, const std::size_t size); + DeviceMem(const DeviceMem&) = delete; + DeviceMem(DeviceMem&& other) + : _size(other._size), + _data(other._data), + _context(other._context), + _log("DeviceMem", Logger::global().level()) { + other._size = 0; + other._data = nullptr; + } + DeviceMem& operator=(const DeviceMem&) = delete; + DeviceMem& operator=(DeviceMem&& other); + + const void* data() const { + return _data; + } + void* data() { + return _data; + } + std::size_t size() const { + return _size; + } + void free(); + ~DeviceMem(); + +private: + std::size_t _size = 0; + void* _data = nullptr; + ze_context_handle_t _context = nullptr; + static const std::size_t _alignment = STANDARD_PAGE_SIZE; + + Logger _log; +}; + +// Create an allocator that uses the ov::Allocator signature that will be used to create the tensor. +class HostMemAllocator final { +public: + explicit HostMemAllocator(const std::shared_ptr& initStructs, + ze_host_mem_alloc_flag_t flag = {}) + : _initStructs(initStructs), + _flag(flag) {} + + /** + * @brief Allocates memory + * @param bytes The size in bytes to allocate + * @return Handle to the allocated resource + */ + void* allocate(const size_t bytes, const size_t alignment = STANDARD_PAGE_SIZE) noexcept; + /** + * @brief Releases handle and all associated memory resources which invalidates the handle. + * @param handle Pointer to allocated data + * @return false if handle cannot be released, otherwise - true. + */ + bool deallocate(void* handle, const size_t bytes, size_t alignment = STANDARD_PAGE_SIZE) noexcept; + + bool is_equal(const HostMemAllocator& other) const; + +private: + const std::shared_ptr _initStructs; + + ze_host_mem_alloc_flag_t _flag; + void* _data = nullptr; + static const std::size_t _alignment = STANDARD_PAGE_SIZE; +}; + +// For graph arguments (inputs and outputs) memory should be located on a host side. For discrete HW +// generation the arguments has to be moved to device side to make it accessible. +// MemoryManagementUnit allow to keeps device allocations in case of discrete HW. +// Usage: we should append graph arguments with corresponding names with `appendArgument` call +// to prepare size statistics and lookup table. To commit memory allocation we should call `allocate` +struct MemoryManagementUnit { + MemoryManagementUnit() = default; + + void appendArgument(const std::string& name, const ze_graph_argument_properties_t& argument); + /* Allocate Device memories */ + void allocate(const ze_device_handle_t device_handle, const ze_context_handle_t context); + + std::size_t getSize() const; + const void* getDeviceMemRegion() const; + void* getDeviceMemRegion(); + + void* getDevicePtr(const std::string& name); + + bool checkHostPtr(const void* ptr) const; + +private: + std::size_t _size = 0; + + std::unique_ptr _device; + std::map _offsets; + + static const std::size_t alignment = STANDARD_PAGE_SIZE; +}; + +} // namespace zeroMemory +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/backend/include/zero_pipeline.hpp b/src/plugins/intel_npu/src/backend/include/zero_pipeline.hpp new file mode 100644 index 00000000000..055fd6deb68 --- /dev/null +++ b/src/plugins/intel_npu/src/backend/include/zero_pipeline.hpp @@ -0,0 +1,38 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "zero_executor.hpp" +#include "zero_memory.hpp" +#include "zero_profiling.hpp" +#include "zero_utils.hpp" +#include "zero_wrappers.hpp" + +namespace intel_npu { +struct Pipeline { +public: + Pipeline() = default; + Pipeline(const Pipeline&) = delete; + Pipeline(Pipeline&&) = delete; + Pipeline& operator=(const Pipeline&) = delete; + Pipeline& operator=(Pipeline&&) = delete; + virtual ~Pipeline() = default; + + virtual void push() = 0; + virtual void pull() = 0; + virtual void reset() const = 0; + +protected: + zeroMemory::MemoryManagementUnit _deviceInputs; + zeroMemory::MemoryManagementUnit _deviceOutputs; +}; + +std::unique_ptr makePipeline(const std::shared_ptr& executorPtr, + const Config& config, + zeroProfiling::ProfilingPool& profiling_pool, + zeroProfiling::ProfilingQuery& profiling_query, + std::shared_ptr npu_profiling, + std::unordered_map>& tensors); +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/backend/include/zero_profiling.hpp b/src/plugins/intel_npu/src/backend/include/zero_profiling.hpp new file mode 100644 index 00000000000..6f559aebe84 --- /dev/null +++ b/src/plugins/intel_npu/src/backend/include/zero_profiling.hpp @@ -0,0 +1,112 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +#include +#include + +#include "intel_npu/al/config/compiler.hpp" +#include "intel_npu/utils/logger/logger.hpp" +#include "openvino/runtime/profiling_info.hpp" + +namespace intel_npu { +namespace zeroProfiling { + +using LayerStatistics = std::vector; + +constexpr uint32_t POOL_SIZE = 1; + +struct ProfilingPool { + ProfilingPool(ze_graph_handle_t graph_handle, + uint32_t profiling_count, + ze_graph_profiling_dditable_ext_t* graph_profiling_ddi_table_ext) + : _graph_handle(graph_handle), + _profiling_count(profiling_count), + _graph_profiling_ddi_table_ext(graph_profiling_ddi_table_ext) {} + ProfilingPool(const ProfilingPool&) = delete; + ProfilingPool& operator=(const ProfilingPool&) = delete; + bool create(); + + ~ProfilingPool(); + + ze_graph_handle_t _graph_handle; + const uint32_t _profiling_count; + ze_graph_profiling_pool_handle_t _handle = nullptr; + ze_graph_profiling_dditable_ext_t* _graph_profiling_ddi_table_ext = nullptr; +}; + +struct ProfilingQuery { + ProfilingQuery(uint32_t index, + ze_device_handle_t device_handle, + ze_graph_profiling_dditable_ext_t* graph_profiling_ddi_table_ext) + : _index(index), + _device_handle(device_handle), + _graph_profiling_ddi_table_ext(graph_profiling_ddi_table_ext) {} + ProfilingQuery(const ProfilingQuery&) = delete; + ProfilingQuery& operator=(const ProfilingQuery&) = delete; + void create(const ze_graph_profiling_pool_handle_t& profiling_pool); + ze_graph_profiling_query_handle_t getHandle() const { + return _handle; + } + LayerStatistics getLayerStatistics() const; + template + std::vector getData() const; + ~ProfilingQuery(); + +private: + void queryGetData(const ze_graph_profiling_type_t profilingType, uint32_t* pSize, uint8_t* pData) const; + void getProfilingProperties(ze_device_profiling_data_properties_t* properties) const; + void verifyProfilingProperties() const; + + const uint32_t _index; + ze_device_handle_t _device_handle; + ze_graph_profiling_query_handle_t _handle = nullptr; + ze_graph_profiling_dditable_ext_t* _graph_profiling_ddi_table_ext = nullptr; +}; + +extern template std::vector ProfilingQuery::getData() const; + +using NpuInferStatistics = std::vector; + +struct NpuInferProfiling final { + explicit NpuInferProfiling(ze_context_handle_t context, ze_device_handle_t device_handle, ov::log::Level loglevel); + NpuInferProfiling(const NpuInferProfiling&) = delete; + NpuInferProfiling& operator=(const NpuInferProfiling&) = delete; + NpuInferProfiling(NpuInferProfiling&&) = delete; + NpuInferProfiling& operator=(NpuInferProfiling&&) = delete; + + void sampleNpuTimestamps(); + NpuInferStatistics getNpuInferStatistics() const; + + ~NpuInferProfiling(); + + /// Buffers allocated by ZE driver + void* npu_ts_infer_start = 0; + void* npu_ts_infer_end = 0; + +private: + ze_context_handle_t _context = nullptr; + ze_device_handle_t _device_handle; + ov::log::Level _loglevel; + Logger _logger; + ze_device_properties_t _dev_properties; + int64_t _npu_infer_stats_min_cc = LLONG_MAX; + int64_t _npu_infer_stats_max_cc = 0; + int64_t _npu_infer_stats_accu_cc = 0; + uint32_t _npu_infer_stats_cnt = 0; + uint32_t _npu_infer_logidx = 0; + static const uint32_t _npu_infer_log_maxsize = 1024; + /// rolling buffer to store duration of last <_npu_infer_log_maxsize number> infers + int64_t _npu_infer_duration_log[_npu_infer_log_maxsize]; + + /// Helper function to convert npu clockcycles to usec + int64_t convertCCtoUS(int64_t val_cc) const; +}; + +} // namespace zeroProfiling +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/backend/include/zero_types.hpp b/src/plugins/intel_npu/src/backend/include/zero_types.hpp new file mode 100644 index 00000000000..50843969863 --- /dev/null +++ b/src/plugins/intel_npu/src/backend/include/zero_types.hpp @@ -0,0 +1,129 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +#include "intel_npu/al/config/runtime.hpp" + +/** + * @brief Last version of Table of Graph Extension functions used within plugin + */ +using ze_graph_dditable_ext_last_t = ze_graph_dditable_ext_1_5_t; + +/** + * @brief Table of Graph Extension functions pointers and function wrappers + * @details Use original Graph Extension functions pointers from driver for function from within lower driver versions. + * Use function wrappers for function from within higher driver versions in order to throw when loaded driver is older + * than required + */ +struct ze_graph_dditable_ext_decorator final { +private: + ze_graph_dditable_ext_last_t* const _impl; + const uint32_t _driverExtVersion; + + ze_graph_dditable_ext_decorator(const ze_graph_dditable_ext_decorator&) = delete; + ze_graph_dditable_ext_decorator(ze_graph_dditable_ext_decorator&&) = delete; + + void throwWhenUnsupported(const std::string func, uint32_t since) { + if (_driverExtVersion < since) { + OPENVINO_THROW("L0 extension function ", + func, + " is only available with driver version ", + ZE_MAJOR_VERSION(since), + ".", + ZE_MINOR_VERSION(since), + " or later"); + } + } + +public: + ze_graph_dditable_ext_decorator(ze_graph_dditable_ext_last_t* impl, uint32_t driverExtVersion) + : _impl(impl), + _driverExtVersion(driverExtVersion) { + // version 1.0 + pfnCreate = _impl->pfnCreate; + pfnDestroy = _impl->pfnDestroy; + pfnGetProperties = _impl->pfnGetProperties; + pfnGetArgumentProperties = _impl->pfnGetArgumentProperties; + pfnSetArgumentValue = _impl->pfnSetArgumentValue; + pfnAppendGraphInitialize = _impl->pfnAppendGraphInitialize; + pfnAppendGraphExecute = _impl->pfnAppendGraphExecute; + pfnGetNativeBinary = _impl->pfnGetNativeBinary; + pfnDeviceGetGraphProperties = _impl->pfnDeviceGetGraphProperties; + + // version 1.1 + pfnGraphGetArgumentMetadata = _impl->pfnGraphGetArgumentMetadata; + pfnGetArgumentProperties2 = _impl->pfnGetArgumentProperties2; + + // version 1.2 + pfnGetArgumentProperties3 = _impl->pfnGetArgumentProperties3; + + // version 1.3 + pfnQueryNetworkCreate = _impl->pfnQueryNetworkCreate; + pfnQueryNetworkDestroy = _impl->pfnQueryNetworkDestroy; + pfnQueryNetworkGetSupportedLayers = _impl->pfnQueryNetworkGetSupportedLayers; + + // version 1.4 + pfnBuildLogGetString = _impl->pfnBuildLogGetString; + + // version 1.5 + // wrappers replace pointers + } + ~ze_graph_dditable_ext_decorator() = default; + + // version 1.0 + ze_pfnGraphCreate_ext_t pfnCreate; + ze_pfnGraphDestroy_ext_t pfnDestroy; + ze_pfnGraphGetProperties_ext_t pfnGetProperties; + ze_pfnGraphGetArgumentProperties_ext_t pfnGetArgumentProperties; + ze_pfnGraphSetArgumentValue_ext_t pfnSetArgumentValue; + ze_pfnAppendGraphInitialize_ext_t pfnAppendGraphInitialize; + ze_pfnAppendGraphExecute_ext_t pfnAppendGraphExecute; + ze_pfnGraphGetNativeBinary_ext_t pfnGetNativeBinary; + ze_pfnDeviceGetGraphProperties_ext_t pfnDeviceGetGraphProperties; + + // version 1.1 + ze_pfnGraphGetArgumentMetadata_ext_t pfnGraphGetArgumentMetadata; + ze_pfnGraphGetArgumentProperties_ext_2_t pfnGetArgumentProperties2; + + // version 1.2 + ze_pfnGraphGetArgumentProperties_ext_3_t pfnGetArgumentProperties3; + + // version 1.3 + ze_pfnGraphQueryNetworkCreate_ext_t pfnQueryNetworkCreate; + ze_pfnGraphQueryNetworkDestroy_ext_t pfnQueryNetworkDestroy; + ze_pfnGraphQueryNetworkGetSupportedLayers_ext_t pfnQueryNetworkGetSupportedLayers; + + // version 1.4 + ze_pfnGraphBuildLogGetString_ext_t pfnBuildLogGetString; + + // version 1.5 + ze_result_t ZE_APICALL pfnCreate2(ze_context_handle_t hContext, + ze_device_handle_t hDevice, + const ze_graph_desc_2_t* desc, + ze_graph_handle_t* phGraph) { + throwWhenUnsupported("pfnCreate2", ZE_GRAPH_EXT_VERSION_1_5); + return _impl->pfnCreate2(hContext, hDevice, desc, phGraph); + } + + ze_result_t ZE_APICALL pfnQueryNetworkCreate2(ze_context_handle_t hContext, + ze_device_handle_t hDevice, + const ze_graph_desc_2_t* desc, + ze_graph_query_network_handle_t* phGraphQueryNetwork) { + throwWhenUnsupported("pfnQueryNetworkCreate2", ZE_GRAPH_EXT_VERSION_1_5); + return _impl->pfnQueryNetworkCreate2(hContext, hDevice, desc, phGraphQueryNetwork); + } + + ze_result_t ZE_APICALL pfnQueryContextMemory(ze_context_handle_t hContext, + ze_graph_memory_query_type_t type, + ze_graph_memory_query_t* query) { + throwWhenUnsupported("pfnQueryContextMemory", ZE_GRAPH_EXT_VERSION_1_5); + return _impl->pfnQueryContextMemory(hContext, type, query); + } +}; + +using ze_graph_dditable_ext_curr_t = ze_graph_dditable_ext_decorator; diff --git a/src/plugins/intel_npu/src/backend/include/zero_utils.hpp b/src/plugins/intel_npu/src/backend/include/zero_utils.hpp new file mode 100644 index 00000000000..3a1dd674200 --- /dev/null +++ b/src/plugins/intel_npu/src/backend/include/zero_utils.hpp @@ -0,0 +1,219 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include + +#include "intel_npu/al/config/runtime.hpp" +#include "intel_npu/utils/logger/logger.hpp" +#include "intel_npu/utils/zero/zero_result.hpp" + +namespace intel_npu { + +namespace zeroUtils { + +static inline void throwOnFail(const std::string& step, const ze_result_t result) { + if (ZE_RESULT_SUCCESS != result) { + OPENVINO_THROW("L0 ", + step, + " result: ", + ze_result_to_string(result), + ", code 0x", + std::hex, + uint64_t(result), + " - ", + ze_result_to_description(result)); + } +} + +static inline void throwOnFail(const std::string& step, const ze_result_t result, const std::string& hintOnError) { + if (ZE_RESULT_SUCCESS != result) { + OPENVINO_THROW("L0 ", + step, + " result: ", + ze_result_to_string(result), + ", code 0x", + std::hex, + uint64_t(result), + " - ", + ze_result_to_description(result), + ". ", + hintOnError); + } +} + +static inline ze_command_queue_priority_t toZeQueuePriority(const ov::hint::Priority& val) { + switch (val) { + case ov::hint::Priority::LOW: + return ZE_COMMAND_QUEUE_PRIORITY_PRIORITY_LOW; + case ov::hint::Priority::MEDIUM: + return ZE_COMMAND_QUEUE_PRIORITY_NORMAL; + case ov::hint::Priority::HIGH: + return ZE_COMMAND_QUEUE_PRIORITY_PRIORITY_HIGH; + default: + OPENVINO_THROW("Incorrect queue priority."); + } +} + +static inline std::size_t precisionToSize(const ze_graph_argument_precision_t val) { + switch (val) { + case ZE_GRAPH_ARGUMENT_PRECISION_INT4: + return 4; + case ZE_GRAPH_ARGUMENT_PRECISION_UINT4: + return 4; + case ZE_GRAPH_ARGUMENT_PRECISION_INT8: + return 8; + case ZE_GRAPH_ARGUMENT_PRECISION_UINT8: + return 8; + case ZE_GRAPH_ARGUMENT_PRECISION_INT16: + return 16; + case ZE_GRAPH_ARGUMENT_PRECISION_UINT16: + return 16; + case ZE_GRAPH_ARGUMENT_PRECISION_INT32: + return 32; + case ZE_GRAPH_ARGUMENT_PRECISION_UINT32: + return 32; + case ZE_GRAPH_ARGUMENT_PRECISION_INT64: + return 64; + case ZE_GRAPH_ARGUMENT_PRECISION_UINT64: + return 64; + case ZE_GRAPH_ARGUMENT_PRECISION_BF16: + return 16; + case ZE_GRAPH_ARGUMENT_PRECISION_FP16: + return 16; + case ZE_GRAPH_ARGUMENT_PRECISION_FP32: + return 32; + case ZE_GRAPH_ARGUMENT_PRECISION_FP64: + return 64; + case ZE_GRAPH_ARGUMENT_PRECISION_BIN: + return 1; + default: + OPENVINO_THROW("precisionToSize switch->default reached"); + } +} + +static inline ze_graph_argument_precision_t getZePrecision(const ov::element::Type_t precision) { + switch (precision) { + case ov::element::Type_t::i4: + return ZE_GRAPH_ARGUMENT_PRECISION_INT4; + case ov::element::Type_t::u4: + return ZE_GRAPH_ARGUMENT_PRECISION_UINT4; + case ov::element::Type_t::i8: + return ZE_GRAPH_ARGUMENT_PRECISION_INT8; + case ov::element::Type_t::u8: + return ZE_GRAPH_ARGUMENT_PRECISION_UINT8; + case ov::element::Type_t::i16: + return ZE_GRAPH_ARGUMENT_PRECISION_INT16; + case ov::element::Type_t::u16: + return ZE_GRAPH_ARGUMENT_PRECISION_UINT16; + case ov::element::Type_t::i32: + return ZE_GRAPH_ARGUMENT_PRECISION_INT32; + case ov::element::Type_t::u32: + return ZE_GRAPH_ARGUMENT_PRECISION_UINT32; + case ov::element::Type_t::i64: + return ZE_GRAPH_ARGUMENT_PRECISION_INT64; + case ov::element::Type_t::u64: + return ZE_GRAPH_ARGUMENT_PRECISION_UINT64; + case ov::element::Type_t::bf16: + return ZE_GRAPH_ARGUMENT_PRECISION_BF16; + case ov::element::Type_t::f16: + return ZE_GRAPH_ARGUMENT_PRECISION_FP16; + case ov::element::Type_t::f32: + return ZE_GRAPH_ARGUMENT_PRECISION_FP32; + case ov::element::Type_t::f64: + return ZE_GRAPH_ARGUMENT_PRECISION_FP64; + case ov::element::Type_t::u1: + return ZE_GRAPH_ARGUMENT_PRECISION_BIN; + default: + return ZE_GRAPH_ARGUMENT_PRECISION_UNKNOWN; + } +} + +static inline std::size_t layoutCount(const ze_graph_argument_layout_t val) { + switch (val) { + case ZE_GRAPH_ARGUMENT_LAYOUT_NCHW: + return 4; + case ZE_GRAPH_ARGUMENT_LAYOUT_NHWC: + return 4; + case ZE_GRAPH_ARGUMENT_LAYOUT_NCDHW: + return 5; + case ZE_GRAPH_ARGUMENT_LAYOUT_NDHWC: + return 5; + case ZE_GRAPH_ARGUMENT_LAYOUT_OIHW: + return 4; + case ZE_GRAPH_ARGUMENT_LAYOUT_C: + return 1; + case ZE_GRAPH_ARGUMENT_LAYOUT_CHW: + return 3; + case ZE_GRAPH_ARGUMENT_LAYOUT_HW: + return 2; + case ZE_GRAPH_ARGUMENT_LAYOUT_NC: + return 2; + case ZE_GRAPH_ARGUMENT_LAYOUT_CN: + return 2; + case ZE_GRAPH_ARGUMENT_LAYOUT_ANY: + // When input has empty shape, val is ZE_GRAPH_ARGUMENT_LAYOUT_ANY + // Add this to pass Single Layer Test on Windows + return 0; + default: + OPENVINO_THROW("layoutCount switch->default reached"); + } +} + +static inline std::size_t getSizeIOBytes(const ze_graph_argument_properties_t& argument) { + std::size_t num_elements = 1; + for (std::size_t i = 0; i < layoutCount(argument.deviceLayout); ++i) { + num_elements *= argument.dims[i]; + } + const std::size_t size_in_bits = num_elements * precisionToSize(argument.devicePrecision); + const std::size_t size_in_bytes = (size_in_bits + (CHAR_BIT - 1)) / CHAR_BIT; + return size_in_bytes; +} + +static inline uint32_t findGroupOrdinal( + const std::vector& command_group_properties, + const ze_device_properties_t& properties) { + auto log = Logger::global().clone("findGroupOrdinal"); + + if (properties.flags & ZE_DEVICE_PROPERTY_FLAG_INTEGRATED) { + for (uint32_t index = 0; index < command_group_properties.size(); ++index) { + const auto& flags = command_group_properties[index].flags; + if ((flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE) != 0 && + (flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COPY) == 0) { + return index; + } + } + + // if we don't find a group where only the proper flag is enabled then search for a group where that flag is + // enabled + for (uint32_t index = 0; index < command_group_properties.size(); ++index) { + const auto& flags = command_group_properties[index].flags; + if (flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE) { + return index; + } + } + + // if still don't find compute flag, return a warning + log.warning("Fail to find a command queue group that contains compute flag, it will be set to 0."); + return 0; + } + + for (uint32_t index = 0; index < command_group_properties.size(); ++index) { + const auto& flags = command_group_properties[index].flags; + if ((flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE) != 0 && + (flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COPY) != 0) { + return index; + } + } + + // if still don't find compute and copy flag, return a warning + log.warning("Fail to find a command queue group that contains compute and copy flags, it will be set to 0."); + return 0; +} + +} // namespace zeroUtils +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/backend/include/zero_wrappers.hpp b/src/plugins/intel_npu/src/backend/include/zero_wrappers.hpp new file mode 100644 index 00000000000..3ec0e402e62 --- /dev/null +++ b/src/plugins/intel_npu/src/backend/include/zero_wrappers.hpp @@ -0,0 +1,155 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +#include "intel_npu/utils/logger/logger.hpp" +#include "zero_types.hpp" +#include "zero_utils.hpp" + +namespace intel_npu { +class CommandList; +class CommandQueue; + +enum stage { + UPLOAD, + EXECUTE, + READBACK, + + COUNT +}; + +class EventPool { +public: + EventPool() = delete; + EventPool(ze_device_handle_t device_handle, + const ze_context_handle_t& context, + uint32_t event_count, + const Config& config); + EventPool(const EventPool&) = delete; + EventPool(EventPool&&) = delete; + EventPool& operator=(const EventPool&) = delete; + EventPool& operator=(EventPool&&) = delete; + ~EventPool(); + inline ze_event_pool_handle_t handle() const { + return _handle; + } + +private: + ze_event_pool_handle_t _handle = nullptr; + + Logger _log; +}; + +class Event { +public: + Event() = delete; + Event(const ze_event_pool_handle_t& event_pool, uint32_t event_index, const Config& config); + Event(const Event&) = delete; + Event(Event&&) = delete; + Event& operator=(const Event&) = delete; + Event& operator=(Event&&) = delete; + + void AppendSignalEvent(CommandList& command_list) const; + void AppendWaitOnEvent(CommandList& command_list); + void AppendEventReset(CommandList& command_list) const; + void hostSynchronize() const; + void reset() const; + ~Event(); + +private: + ze_event_handle_t _handle = nullptr; + + Logger _log; +}; + +class CommandList { +public: + friend class CommandQueue; + CommandList() = delete; + CommandList(const ze_device_handle_t& device_handle, + const ze_context_handle_t& context, + ze_graph_dditable_ext_curr_t* graph_ddi_table_ext, + const Config& config, + const uint32_t& group_ordinal); + CommandList(const CommandList&) = delete; + CommandList(CommandList&&) = delete; + CommandList& operator=(const CommandList&) = delete; + CommandList& operator=(CommandList&&) = delete; + + void reset() const; + void appendMemoryCopy(void* dst, const void* src, const std::size_t size) const; + void appendGraphInitialize(const ze_graph_handle_t& graph_handle) const; + void appendGraphExecute(const ze_graph_handle_t& graph_handle, + const ze_graph_profiling_query_handle_t& profiling_query_handle) const; + void appendNpuTimestamp(uint64_t* timestamp_buff) const; + void appendBarrier() const; + void close() const; + ~CommandList(); + + inline ze_command_list_handle_t handle() const { + return _handle; + } + +private: + ze_command_list_handle_t _handle = nullptr; + const ze_context_handle_t _context = nullptr; + ze_graph_dditable_ext_curr_t* _graph_ddi_table_ext = nullptr; + + Logger _log; +}; + +class Fence { +public: + Fence() = delete; + Fence(const CommandQueue& command_queue, const Config& config); + Fence(const Fence&) = delete; + Fence(Fence&&) = delete; + Fence& operator=(const Fence&) = delete; + Fence& operator=(Fence&&) = delete; + + void reset() const; + void hostSynchronize() const; + ~Fence(); + inline ze_fence_handle_t handle() const { + return _handle; + } + +private: + ze_fence_handle_t _handle = nullptr; + + Logger _log; +}; + +class CommandQueue { +public: + CommandQueue() = delete; + CommandQueue(const ze_device_handle_t& device_handle, + const ze_context_handle_t& context, + const ze_command_queue_priority_t& priority, + const Config& config, + const uint32_t& group_ordinal); + CommandQueue(const CommandQueue&) = delete; + CommandQueue(CommandQueue&&) = delete; + CommandQueue& operator=(const CommandQueue&) = delete; + CommandQueue& operator=(CommandQueue&&) = delete; + + void executeCommandList(CommandList& command_list) const; + void executeCommandList(CommandList& command_list, Fence& fence) const; + ~CommandQueue(); + inline ze_command_queue_handle_t handle() const { + return _handle; + } + +private: + ze_command_queue_handle_t _handle = nullptr; + ze_context_handle_t _context = nullptr; + + Logger _log; +}; + +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/backend/src/zero_backend.cpp b/src/plugins/intel_npu/src/backend/src/zero_backend.cpp new file mode 100644 index 00000000000..facedd78998 --- /dev/null +++ b/src/plugins/intel_npu/src/backend/src/zero_backend.cpp @@ -0,0 +1,46 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "zero_backend.hpp" + +#include + +#include "intel_npu/al/config/common.hpp" +#include "zero_device.hpp" + +namespace intel_npu { + +ZeroEngineBackend::ZeroEngineBackend(const Config& config) { + Logger::global().setLevel(config.get()); + + _instance = std::make_shared(); + + auto device = std::make_shared(_instance); + _devices.emplace(std::make_pair(device->getName(), device)); +} + +ZeroEngineBackend::~ZeroEngineBackend() = default; + +const std::shared_ptr ZeroEngineBackend::getDevice() const { + if (_devices.empty()) { + return {}; + } else { + return _devices.begin()->second; + } +} + +const std::shared_ptr ZeroEngineBackend::getDevice(const std::string& /*name*/) const { + // TODO Add the search of the device by platform & slice + return getDevice(); +} + +const std::vector ZeroEngineBackend::getDeviceNames() const { + std::vector devicesNames; + std::for_each(_devices.cbegin(), _devices.cend(), [&devicesNames](const auto& device) { + devicesNames.push_back(device.first); + }); + return devicesNames; +} + +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/backend/src/zero_device.cpp b/src/plugins/intel_npu/src/backend/src/zero_device.cpp new file mode 100644 index 00000000000..dc0c8041a3b --- /dev/null +++ b/src/plugins/intel_npu/src/backend/src/zero_device.cpp @@ -0,0 +1,125 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "zero_device.hpp" + +#include + +#include "intel_npu/al/itt.hpp" +#include "zero_executor.hpp" +#include "zero_infer_request.hpp" + +using namespace intel_npu; + +ZeroDevice::ZeroDevice(const std::shared_ptr& initStructs) + : _initStructs(initStructs), + _graph_ddi_table_ext(_initStructs->getGraphDdiTable()), + log("ZeroDevice", Logger::global().level()) { + device_properties.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; + zeroUtils::throwOnFail("zeDeviceGetProperties", + zeDeviceGetProperties(_initStructs->getDevice(), &device_properties)); + driver_properties.stype = ZE_STRUCTURE_TYPE_DRIVER_PROPERTIES; + zeroUtils::throwOnFail("zeDriverGetProperties", + zeDriverGetProperties(_initStructs->getDriver(), &driver_properties)); + + std::vector command_group_properties; + uint32_t command_queue_group_count = 0; + // Discover all command queue groups + zeroUtils::throwOnFail( + "zeDeviceGetCommandQueueGroupProperties", + zeDeviceGetCommandQueueGroupProperties(_initStructs->getDevice(), &command_queue_group_count, nullptr)); + + command_group_properties.resize(command_queue_group_count); + + for (auto& prop : command_group_properties) { + prop.stype = ZE_STRUCTURE_TYPE_COMMAND_QUEUE_GROUP_PROPERTIES; + prop.pNext = nullptr; + } + + zeroUtils::throwOnFail("zeDeviceGetCommandQueueGroupProperties", + zeDeviceGetCommandQueueGroupProperties(_initStructs->getDevice(), + &command_queue_group_count, + command_group_properties.data())); + + // Find the corresponding command queue group. + _group_ordinal = zeroUtils::findGroupOrdinal(command_group_properties, device_properties); +} + +std::shared_ptr ZeroDevice::createExecutor( + const std::shared_ptr& networkDescription, + const Config& config) { + OV_ITT_SCOPED_TASK(itt::domains::LevelZeroBackend, "Device::createExecutor"); + return std::make_shared(_initStructs, networkDescription, config, _group_ordinal); +} + +std::string ZeroDevice::getName() const { +// KMD is setting usDeviceID from VpuFamilyID.h +#define NPU_3700_DEVICE_ID 0x6240 +#define NPU_3720_P_DEVICE_ID 0x7D1D +#define NPU_3720_S_DEVICE_ID 0xAD1D + + std::string name; + switch (device_properties.deviceId) { + case NPU_3700_DEVICE_ID: + name = "3700"; + break; + case NPU_3720_P_DEVICE_ID: + case NPU_3720_S_DEVICE_ID: + name = ov::intel_npu::Platform::NPU3720; + break; + default: + name = "AUTO_DETECT"; + } + + return name; +} + +std::string ZeroDevice::getFullDeviceName() const { + return device_properties.name; +} + +IDevice::Uuid ZeroDevice::getUuid() const { + Uuid uuid{}; + static_assert(sizeof(device_properties.uuid.id) == uuid.uuid.size(), + "ze_device_uuid_t::id size doesn't match intel_npu::Uuid::uuid size"); + + std::copy(std::begin(device_properties.uuid.id), std::end(device_properties.uuid.id), std::begin(uuid.uuid)); + + return uuid; +} + +uint32_t ZeroDevice::getDriverVersion() const { + return driver_properties.driverVersion; +} + +uint32_t ZeroDevice::getSubDevId() const { + return device_properties.subdeviceId; +} + +uint32_t ZeroDevice::getMaxNumSlices() const { + return device_properties.numSlices; +} + +uint64_t ZeroDevice::getAllocMemSize() const { + ze_graph_memory_query_t query{}; + zeroUtils::throwOnFail( + "pfnQueryContextMemory", + _graph_ddi_table_ext->pfnQueryContextMemory(_initStructs->getContext(), ZE_GRAPH_QUERY_MEMORY_DDR, &query)); + return query.allocated; +} + +uint64_t ZeroDevice::getTotalMemSize() const { + ze_graph_memory_query_t query{}; + zeroUtils::throwOnFail( + "pfnQueryContextMemory", + _graph_ddi_table_ext->pfnQueryContextMemory(_initStructs->getContext(), ZE_GRAPH_QUERY_MEMORY_DDR, &query)); + return query.total; +} + +std::shared_ptr ZeroDevice::createInferRequest( + const std::shared_ptr& compiledModel, + const std::shared_ptr& executor, + const Config& config) { + return std::make_shared(_initStructs, compiledModel, executor, config); +} diff --git a/src/plugins/intel_npu/src/backend/src/zero_executor.cpp b/src/plugins/intel_npu/src/backend/src/zero_executor.cpp new file mode 100644 index 00000000000..0ce68bf4a0e --- /dev/null +++ b/src/plugins/intel_npu/src/backend/src/zero_executor.cpp @@ -0,0 +1,105 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "zero_executor.hpp" + +#include + +#include +#include +#include +#include + +#include "intel_npu/al/config/common.hpp" +#include "intel_npu/al/itt.hpp" +#include "zero_device.hpp" +#include "zero_utils.hpp" + +using namespace intel_npu; + +ZeroExecutor::ZeroExecutor(const std::shared_ptr& initStructs, + const std::shared_ptr& networkDescription, + const Config& config, + const uint32_t& group_ordinal) + : _config(config), + _logger("Graph", _config.get()), + _initStructs(initStructs), + _networkDesc(networkDescription), + _graph_ddi_table_ext(_initStructs->getGraphDdiTable()), + _group_ordinal(group_ordinal), + _command_queues{{std::make_shared(_initStructs->getDevice(), + _initStructs->getContext(), + zeroUtils::toZeQueuePriority(_config.get()), + _config, + group_ordinal), + std::make_shared(_initStructs->getDevice(), + _initStructs->getContext(), + zeroUtils::toZeQueuePriority(_config.get()), + _config, + group_ordinal), + std::make_shared(_initStructs->getDevice(), + _initStructs->getContext(), + zeroUtils::toZeQueuePriority(_config.get()), + _config, + group_ordinal)}} { + OV_ITT_SCOPED_TASK(itt::domains::LevelZeroBackend, "Executor::ZeroExecutor"); + CommandList graph_command_list(_initStructs->getDevice(), + _initStructs->getContext(), + _initStructs->getGraphDdiTable(), + _config, + _group_ordinal); + CommandQueue graph_command_queue(_initStructs->getDevice(), + _initStructs->getContext(), + ZE_COMMAND_QUEUE_PRIORITY_NORMAL, + _config, + _group_ordinal); + Fence fence(graph_command_queue, _config); + ze_device_properties_t properties = {}; + properties.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; + zeroUtils::throwOnFail("zeDeviceGetProperties", zeDeviceGetProperties(_initStructs->getDevice(), &properties)); + + OV_ITT_TASK_CHAIN(ZERO_EXECUTOR_GRAPH, itt::domains::LevelZeroBackend, "Executor::ZeroExecutor", "graphCreate"); + ze_graph_desc_t desc{ZE_STRUCTURE_TYPE_GRAPH_DESC_PROPERTIES, + nullptr, + ZE_GRAPH_FORMAT_NATIVE, + _networkDesc->compiledNetwork.size(), + _networkDesc->compiledNetwork.data(), + nullptr}; + zeroUtils::throwOnFail( + "pfnCreate", + _graph_ddi_table_ext->pfnCreate(_initStructs->getContext(), _initStructs->getDevice(), &desc, &_graph)); + + OV_ITT_TASK_NEXT(ZERO_EXECUTOR_GRAPH, "pfnGetProperties"); + zeroUtils::throwOnFail("pfnGetProperties", _graph_ddi_table_ext->pfnGetProperties(_graph, &_props)); + + OV_ITT_TASK_NEXT(ZERO_EXECUTOR_GRAPH, "pfnGetArgumentProperties"); + for (uint32_t index = 0; index < _props.numGraphArgs; ++index) { + ze_graph_argument_properties_t arg; + zeroUtils::throwOnFail("pfnGetArgumentProperties", + _graph_ddi_table_ext->pfnGetArgumentProperties(_graph, index, &arg)); + if (ZE_GRAPH_ARGUMENT_TYPE_INPUT == arg.type) { + _inputs_desc_map.emplace(std::make_pair(std::string(arg.name), ArgumentDescriptor{arg, index})); + } else { + _outputs_desc_map.emplace(std::make_pair(std::string(arg.name), ArgumentDescriptor{arg, index})); + } + } + OV_ITT_TASK_NEXT(ZERO_EXECUTOR_GRAPH, "appendGraphInitialize"); + graph_command_list.appendGraphInitialize(_graph); + graph_command_list.close(); + + OV_ITT_TASK_NEXT(ZERO_EXECUTOR_GRAPH, "queue_execute"); + graph_command_queue.executeCommandList(graph_command_list, fence); + fence.hostSynchronize(); +} + +void ZeroExecutor::setArgumentValue(uint32_t argi_, const void* argv_) const { + zeroUtils::throwOnFail("zeGraphSetArgumentValue", _graph_ddi_table_ext->pfnSetArgumentValue(_graph, argi_, argv_)); +} + +ZeroExecutor::~ZeroExecutor() { + auto result = _graph_ddi_table_ext->pfnDestroy(_graph); + if (ZE_RESULT_SUCCESS != result) { + _logger.error("_graph_ddi_table_ext->pfnDestroy failed %#X", uint64_t(result)); + } +} diff --git a/src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp b/src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp new file mode 100644 index 00000000000..e1df23b3c9a --- /dev/null +++ b/src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp @@ -0,0 +1,264 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "zero_infer_request.hpp" + +#include "intel_npu/al/config/common.hpp" +#include "intel_npu/al/config/compiler.hpp" +#include "intel_npu/al/config/runtime.hpp" +#include "intel_npu/al/itt.hpp" +#include "intel_npu/al/prefix.hpp" +#include "zero_memory.hpp" + +using namespace intel_npu; + +namespace { + +constexpr bool STATE_TENSOR = true; +constexpr bool NOT_STATE_TENSOR = false; + +/** + * @brief Checks that the metadata of the provided descriptor corresponds to the values registered in the Level Zero + * structure. + * @param nodeDescriptor The OpenVINO API specific I/O descriptor which shall be compared. + * @param zeDescriptor The Level Zero specific structure used for comparison. + * @param name Tensor identifier used for error logging. + */ +void check_level_zero_attributes_match(const IONodeDescriptor& nodeDescriptor, + const ZeroExecutor::ArgumentDescriptor& zeDescriptor, + const std::string& name) { + const ov::element::Type_t ovPrecision = nodeDescriptor.precision; + const ze_graph_argument_precision_t zePrecision = zeDescriptor.info.devicePrecision; + + if (zeroUtils::getZePrecision(ovPrecision) != zePrecision) { + OPENVINO_THROW("Precision mismatch for parameter " + name); + } + + const std::vector& ovDimensions = nodeDescriptor.originalShape.get_shape(); + + if (ovDimensions.size() > ZE_MAX_GRAPH_ARGUMENT_DIMENSIONS_SIZE) { + OPENVINO_THROW( + "Maximum number of dimensions supported: " + std::to_string(ZE_MAX_GRAPH_ARGUMENT_DIMENSIONS_SIZE) + '\n' + + "Given: " + std::to_string(ovDimensions.size())); + } + + for (size_t index = 0; index < ovDimensions.size(); ++index) { + if (ovDimensions[index] != zeDescriptor.info.dims[index]) { + OPENVINO_THROW("Shape mismatch for parameter " + name); + } + } + for (size_t index = ovDimensions.size(); index < ZE_MAX_GRAPH_ARGUMENT_DIMENSIONS_SIZE; ++index) { + if (zeDescriptor.info.dims[index] != 0 && zeDescriptor.info.dims[index] != 1) { + OPENVINO_THROW("Shape mismatch for parameter " + name); + } + } +} + +} // namespace + +//------------------------------------------------------------------------------ +ZeroInferRequest::ZeroInferRequest(const std::shared_ptr& backendPtr, + const std::shared_ptr& compiledModel, + const std::shared_ptr& executor, + const Config& config) + : SyncInferRequest(compiledModel), + _executorPtr(executor), + _executor(static_cast(_executorPtr.get())), + _config(config), + _logger("ZeroInferRequest", config.get()), + _profiling_pool(_executor->graph(), + zeroProfiling::POOL_SIZE, + _executor->getInitStructs()->getProfilingDdiTable()), + _profiling_query(0, + _executor->getInitStructs()->getDevice(), + _executor->getInitStructs()->getProfilingDdiTable()) { + const std::unordered_map& executorInputDescriptors = + _executor->inputs_desc_map(); + const std::unordered_map& executorOutputDescriptors = + _executor->outputs_desc_map(); + + auto proftype = config.get(); + if (proftype == ov::intel_npu::ProfilingType::INFER) { + _npu_profiling = std::make_shared(_executor->getInitStructs()->getContext(), + _executor->getInitStructs()->getDevice(), + _config.get()); + } + + ze_device_properties_t properties = {}; + properties.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; + zeroUtils::throwOnFail("zeDeviceGetProperties", + zeDeviceGetProperties(_executor->getInitStructs()->getDevice(), &properties)); + + for (const std::string& inputName : _metadata.inputNames) { + if (!executorInputDescriptors.count(inputName)) { + OPENVINO_THROW("Invalid graph input descriptor key: " + inputName); + } + + const IONodeDescriptor& parameterDescriptor = _metadata.parameters.at(inputName); + check_level_zero_attributes_match(parameterDescriptor, executorInputDescriptors.at(inputName), inputName); + + ov::Allocator allocator; + if (properties.flags & ZE_DEVICE_PROPERTY_FLAG_INTEGRATED) { + allocator = zeroMemory::HostMemAllocator(backendPtr, ZE_HOST_MEM_ALLOC_FLAG_BIAS_WRITE_COMBINED); + } else { + allocator = zeroMemory::HostMemAllocator(backendPtr); + } + + // The I/O buffers already allocated using the Level Zero API are being reused here + allocate_tensor(inputName, parameterDescriptor, NOT_STATE_TENSOR, allocator); + } + + for (const std::string& outputName : _metadata.outputNames) { + if (!executorOutputDescriptors.count(outputName)) { + OPENVINO_THROW("Invalid graph output descriptor key: " + outputName); + } + + const IONodeDescriptor& resultDescriptor = _metadata.results.at(outputName); + check_level_zero_attributes_match(resultDescriptor, executorOutputDescriptors.at(outputName), outputName); + + auto allocator = zeroMemory::HostMemAllocator(backendPtr); + + allocate_tensor(outputName, resultDescriptor, NOT_STATE_TENSOR, allocator); + } + + for (const std::string& stateName : _metadata.stateNames) { + const std::string& stateInputBufferName = READVALUE_PREFIX + stateName; + const std::string& stateOutputBufferName = ASSIGN_PREFIX + stateName; + + if (!executorInputDescriptors.count(stateInputBufferName)) { + OPENVINO_THROW("Invalid graph input descriptor key: " + stateInputBufferName); + } + if (!executorOutputDescriptors.count(stateOutputBufferName)) { + OPENVINO_THROW("Invalid graph output descriptor key: " + stateOutputBufferName); + } + + const IONodeDescriptor& stateDescriptor = _metadata.states.at(stateName); + check_level_zero_attributes_match(stateDescriptor, + executorInputDescriptors.at(stateInputBufferName), + stateInputBufferName); + check_level_zero_attributes_match(stateDescriptor, + executorOutputDescriptors.at(stateOutputBufferName), + stateOutputBufferName); + + auto allocator = zeroMemory::HostMemAllocator(backendPtr); + + // Only one buffer per state variable is required, we'll use the "output" one since this one captures the latest + // tensor value + allocate_tensor(stateName, stateDescriptor, STATE_TENSOR, allocator); + } + + /// Construct pipepline + _pipeline = makePipeline(_executorPtr, _config, _profiling_pool, _profiling_query, _npu_profiling, _copyAllTensors); +} + +void ZeroInferRequest::infer() { + infer_async(); + get_result(); +} + +void ZeroInferRequest::infer_async() { + _logger.debug("InferRequest::infer_async started"); + OV_ITT_SCOPED_TASK(itt::domains::LevelZeroBackend, "infer_async"); + + for (const auto& name : _inputAndStateInputNames) { + const std::shared_ptr& inputTensor = _allTensors.at(name); + const std::shared_ptr& wrapperInputTensor = _copyAllTensors.at(name); + + const uint8_t* tensorBuffer = reinterpret_cast(inputTensor->data()); + uint8_t* copyTensorBuffer = reinterpret_cast(wrapperInputTensor->data()); + + if (tensorBuffer != copyTensorBuffer) { + if (tensorBuffer == nullptr || copyTensorBuffer == nullptr) { + OPENVINO_THROW("Empty buffer"); + } + + std::memcpy(copyTensorBuffer, tensorBuffer, inputTensor->get_byte_size()); + } + } + + _pipeline->push(); +} + +void ZeroInferRequest::get_result() { + OV_ITT_SCOPED_TASK(itt::domains::LevelZeroBackend, "get_result"); + + _pipeline->pull(); + + for (const auto& name : _outputAndStateOutputNames) { + const std::shared_ptr& outputTensor = _allTensors.at(name); + const std::shared_ptr& wrapperOutputTensor = _copyAllTensors.at(name); + + uint8_t* tensorBuffer = reinterpret_cast(outputTensor->data()); + const uint8_t* copyTensorBuffer = reinterpret_cast(wrapperOutputTensor->data()); + + if (tensorBuffer != copyTensorBuffer) { + if (tensorBuffer == nullptr || copyTensorBuffer == nullptr) { + OPENVINO_THROW("Empty buffer"); + } + + std::memcpy(tensorBuffer, copyTensorBuffer, outputTensor->get_byte_size()); + } + } + + _pipeline->reset(); + _logger.debug("InferRequest::get_result finished"); +} + +void ZeroInferRequest::check_network_precision(const ov::element::Type_t precision) { + switch (precision) { + case ov::element::Type_t::f32: + break; + case ov::element::Type_t::f16: + break; + case ov::element::Type_t::u8: + break; + case ov::element::Type_t::i8: + break; + case ov::element::Type_t::u16: + break; + case ov::element::Type_t::i16: + break; + case ov::element::Type_t::u32: + break; + case ov::element::Type_t::i32: + break; + case ov::element::Type_t::u64: + break; + case ov::element::Type_t::i64: + break; + default: + OPENVINO_THROW("Unsupported tensor precision: " + ov::element::Type(precision).get_type_name() + + "! Supported precisions: FP32, FP16, U8, I8, U16, I16, U32, I32, U64, I64"); + } +} + +std::vector ZeroInferRequest::get_profiling_info() const { + const auto& compiledModel = *std::dynamic_pointer_cast(_compiledModel); + const auto& compilerConfig = compiledModel.get_config(); + if (!compilerConfig.get() || !_config.get()) { + return {}; + } + + auto compilerType = compilerConfig.get(); + if (compilerType == ov::intel_npu::CompilerType::MLIR) { + // For plugin compiler retreive raw profiling data from backend and delegate + // processing to the compiler + const auto& networkDesc = compiledModel.get_network_description(); + const auto& compiler = compiledModel.get_compiler(); + const auto& blob = networkDesc->compiledNetwork; + auto profData = get_raw_profiling_data(); + return compiler->process_profiling_output(profData, blob, compilerConfig); + } else { + auto proftype = _config.get(); + if (proftype == ov::intel_npu::ProfilingType::INFER) { + return _npu_profiling->getNpuInferStatistics(); + } else { /// proftype = MODEL or undefined = fallback to model profiling + return _profiling_query.getLayerStatistics(); + } + } +} + +std::vector ZeroInferRequest::get_raw_profiling_data() const { + return _profiling_query.getData(); +} diff --git a/src/plugins/intel_npu/src/backend/src/zero_init.cpp b/src/plugins/intel_npu/src/backend/src/zero_init.cpp new file mode 100644 index 00000000000..1cff12491b6 --- /dev/null +++ b/src/plugins/intel_npu/src/backend/src/zero_init.cpp @@ -0,0 +1,152 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "zero_init.hpp" + +#include "intel_npu/al/itt.hpp" +#include "zero_utils.hpp" + +namespace intel_npu { + +const ze_driver_uuid_t ZeroInitStructsHolder::uuid = ze_intel_vpu_driver_uuid; + +static std::tuple queryDriverExtensionVersion(ze_driver_handle_t _driverHandle) { + // query the extension properties + uint32_t count = 0; + zeroUtils::throwOnFail("zeDriverGetExtensionProperties", + zeDriverGetExtensionProperties(_driverHandle, &count, nullptr)); + + std::vector extProps; + extProps.resize(count); + zeroUtils::throwOnFail("zeDriverGetExtensionProperties", + zeDriverGetExtensionProperties(_driverHandle, &count, extProps.data())); + + const char* graphExtName = nullptr; + uint32_t targetVersion = 0; + for (uint32_t i = 0; i < count; ++i) { + auto& property = extProps[i]; + + if (strncmp(property.name, ZE_GRAPH_EXT_NAME, strlen(ZE_GRAPH_EXT_NAME)) != 0) + continue; + + // If the driver version is latest, will just use its name. + if (property.version == ZE_GRAPH_EXT_VERSION_CURRENT) { + graphExtName = property.name; + targetVersion = property.version; + break; + } + + // Use the latest version supported by the driver. + if (property.version > targetVersion) { + graphExtName = property.name; + targetVersion = property.version; + } + } + + if (graphExtName == nullptr) { + OPENVINO_THROW("queryDriverExtensionVersion: Failed to find Graph extension in NPU Driver"); + } + + const uint16_t supportedDriverExtMajorVersion = 1; + const uint16_t driverExtMajorVersion = ZE_MAJOR_VERSION(targetVersion); + if (supportedDriverExtMajorVersion != driverExtMajorVersion) { + OPENVINO_THROW("Plugin supports only driver with extension major version ", + supportedDriverExtMajorVersion, + "; discovered driver extension has major version ", + driverExtMajorVersion); + } + + return std::make_tuple(targetVersion, graphExtName); +} + +ZeroInitStructsHolder::ZeroInitStructsHolder() : log("NPUZeroInitStructsHolder", Logger::global().level()) { + OV_ITT_SCOPED_TASK(itt::domains::LevelZeroBackend, "ZeroInitStructsHolder::ZeroInitStructsHolder"); + zeroUtils::throwOnFail("zeInit", zeInit(ZE_INIT_FLAG_VPU_ONLY)); + + uint32_t drivers = 0; + zeroUtils::throwOnFail("zeDriverGet", zeDriverGet(&drivers, nullptr)); + + std::vector all_drivers(drivers); + zeroUtils::throwOnFail("zeDriverGet", zeDriverGet(&drivers, all_drivers.data())); + + // Get our target driver + ze_driver_properties_t props = {}; + props.stype = ZE_STRUCTURE_TYPE_DRIVER_PROPERTIES; + for (uint32_t i = 0; i < drivers; ++i) { + zeDriverGetProperties(all_drivers[i], &props); + + if (memcmp(&props.uuid, &uuid, sizeof(uuid)) == 0) { + driver_handle = all_drivers[i]; + break; + } + } + if (driver_handle == nullptr) { + OPENVINO_THROW("zeDriverGet failed to return NPU driver"); + } + + // Check L0 API version + ze_api_version_t ze_drv_api_version = {}; + zeroUtils::throwOnFail("zeDriverGetApiVersion", zeDriverGetApiVersion(driver_handle, &ze_drv_api_version)); + + if (ZE_MAJOR_VERSION(ZE_API_VERSION_CURRENT) != ZE_MAJOR_VERSION(ze_drv_api_version)) { + OPENVINO_THROW("Incompatibility between NPU plugin and driver! ", + "Plugin L0 API major version = ", + ZE_MAJOR_VERSION(ZE_API_VERSION_CURRENT), + ", ", + "Driver L0 API major version = ", + ZE_MAJOR_VERSION(ze_drv_api_version)); + } + if (ZE_MINOR_VERSION(ZE_API_VERSION_CURRENT) != ZE_MINOR_VERSION(ze_drv_api_version)) { + log.debug("Some features might not be available! " + "Plugin L0 API minor version = %d, Driver L0 API minor version = %d", + ZE_MINOR_VERSION(ZE_API_VERSION_CURRENT), + ZE_MINOR_VERSION(ze_drv_api_version)); + } + + // Query our graph extension version + uint32_t driverExtVersion = 0; + std::string graphExtName; + std::tie(driverExtVersion, graphExtName) = queryDriverExtensionVersion(driver_handle); + + log.debug("Found Driver Version %d.%d, Driver Extension Version %d.%d (%s)", + ZE_MAJOR_VERSION(ze_drv_api_version), + ZE_MINOR_VERSION(ze_drv_api_version), + ZE_MAJOR_VERSION(driverExtVersion), + ZE_MINOR_VERSION(driverExtVersion), + graphExtName.c_str()); + + // Load our graph extension + ze_graph_dditable_ext_last_t* graph_ddi_table_ext = nullptr; + zeroUtils::throwOnFail("zeDriverGetExtensionFunctionAddress", + zeDriverGetExtensionFunctionAddress(driver_handle, + graphExtName.c_str(), + reinterpret_cast(&graph_ddi_table_ext))); + graph_dditable_ext_decorator = + std::make_unique(graph_ddi_table_ext, driverExtVersion); + + // Load our profiling extension + zeroUtils::throwOnFail( + "zeDriverGetExtensionFunctionAddress", + zeDriverGetExtensionFunctionAddress(driver_handle, + "ZE_extension_profiling_data", + reinterpret_cast(&_graph_profiling_ddi_table_ext))); + + uint32_t device_count = 1; + // Get our target device + zeroUtils::throwOnFail("zeDeviceGet", zeDeviceGet(driver_handle, &device_count, &device_handle)); + + ze_context_desc_t context_desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, 0, 0}; + zeroUtils::throwOnFail("zeContextCreate", zeContextCreate(driver_handle, &context_desc, &context)); +} + +ZeroInitStructsHolder::~ZeroInitStructsHolder() { + if (context) { + auto result = zeContextDestroy(context); + if (ZE_RESULT_SUCCESS != result) { + log.error("zeContextDestroy failed %#X", uint64_t(result)); + } + } +} + +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/backend/src/zero_memory.cpp b/src/plugins/intel_npu/src/backend/src/zero_memory.cpp new file mode 100644 index 00000000000..d92e817f1cd --- /dev/null +++ b/src/plugins/intel_npu/src/backend/src/zero_memory.cpp @@ -0,0 +1,108 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "zero_memory.hpp" + +#include "zero_utils.hpp" + +namespace intel_npu { +namespace zeroMemory { +DeviceMem::DeviceMem(const ze_device_handle_t device_handle, ze_context_handle_t context, const std::size_t size) + : _size(size), + _context(context), + _log("DeviceMem", Logger::global().level()) { + ze_device_mem_alloc_desc_t desc = {ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, nullptr, 0, 0}; + + zeroUtils::throwOnFail("zeMemAllocDevice", + zeMemAllocDevice(_context, &desc, _size, _alignment, device_handle, &_data)); +} +DeviceMem& DeviceMem::operator=(DeviceMem&& other) { + if (this == &other) + return *this; + free(); + _size = other._size; + _data = other._data; + _context = other._context; + other._size = 0; + other._data = nullptr; + return *this; +} +void DeviceMem::free() { + if (_size != 0) { + _size = 0; + zeroUtils::throwOnFail("zeMemFree DeviceMem", zeMemFree(_context, _data)); + _data = nullptr; + } +} +DeviceMem::~DeviceMem() { + try { + free(); + } catch (const std::exception& e) { + _log.error("Caught when freeing memory: %s", e.what()); + } +} + +void* HostMemAllocator::allocate(const size_t bytes, const size_t /*alignment*/) noexcept { + size_t size = bytes + _alignment - (bytes % _alignment); + + ze_host_mem_alloc_desc_t desc = {ZE_STRUCTURE_TYPE_HOST_MEM_ALLOC_DESC, + nullptr, + static_cast(_flag)}; + ze_result_t res = zeMemAllocHost(_initStructs->getContext(), &desc, size, _alignment, &_data); + + if (res == ZE_RESULT_SUCCESS) { + return _data; + } else { + return nullptr; + } +} +bool HostMemAllocator::deallocate(void* handle, const size_t /* bytes */, size_t /* alignment */) noexcept { + auto result = zeMemFree(_initStructs->getContext(), handle); + if (ZE_RESULT_SUCCESS != result) { + return false; + } + + return true; +} +bool HostMemAllocator::is_equal(const HostMemAllocator& other) const { + return other._data != nullptr && _data != nullptr && other._data == _data; +} + +void MemoryManagementUnit::appendArgument(const std::string& name, const ze_graph_argument_properties_t& argument) { + _offsets.emplace(std::make_pair(name, _size)); + + const std::size_t argSize = zeroUtils::getSizeIOBytes(argument); + _size += argSize + alignment - + (argSize % alignment); // is this really necessary? if 0==argSize%alignment -> add 1 * alignment +} + +void MemoryManagementUnit::allocate(const ze_device_handle_t device_handle, const ze_context_handle_t context) { + if (_size == 0) { + OPENVINO_THROW("Can't allocate empty buffer"); + } + + _device = std::make_unique(device_handle, context, _size); +} +std::size_t MemoryManagementUnit::getSize() const { + return _size; +} +const void* MemoryManagementUnit::getDeviceMemRegion() const { + return _device ? _device->data() : nullptr; +} +void* MemoryManagementUnit::getDeviceMemRegion() { + return _device ? _device->data() : nullptr; +} +void* MemoryManagementUnit::getDevicePtr(const std::string& name) { + uint8_t* from = static_cast(_device ? _device->data() : nullptr); + if (from == nullptr) { + OPENVINO_THROW("Device memory not allocated yet"); + } + if (!_offsets.count(name)) { + OPENVINO_THROW("Invalid memory offset key: ", name); + } + + return _offsets.at(name) + from; +} +} // namespace zeroMemory +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/backend/src/zero_pipeline.cpp b/src/plugins/intel_npu/src/backend/src/zero_pipeline.cpp new file mode 100644 index 00000000000..531b3a087a2 --- /dev/null +++ b/src/plugins/intel_npu/src/backend/src/zero_pipeline.cpp @@ -0,0 +1,288 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "zero_pipeline.hpp" + +#include +#include + +#include "intel_npu/al/itt.hpp" +#include "intel_npu/al/prefix.hpp" +#include "intel_npu/utils/logger/logger.hpp" + +namespace intel_npu { + +struct DiscretePipeline final : public Pipeline { +public: + DiscretePipeline(const Config& config, + const ze_device_handle_t& device_handle, + ze_context_handle_t& context, + ze_graph_dditable_ext_curr_t* graph_ddi_table_ext, + const std::shared_ptr& executorPtr, + ze_graph_profiling_query_handle_t profiling_handle, + const std::array, stage::COUNT>& command_queues, + const uint32_t& group_ordinal, + std::unordered_map>& tensors) + : _config(config), + _command_queues{command_queues}, + _command_list{{{device_handle, context, graph_ddi_table_ext, _config, group_ordinal}, + {device_handle, context, graph_ddi_table_ext, _config, group_ordinal}, + {device_handle, context, graph_ddi_table_ext, _config, group_ordinal}}}, + _fence{{{*_command_queues[stage::UPLOAD], _config}, + {*_command_queues[stage::EXECUTE], _config}, + {*_command_queues[stage::READBACK], _config}}}, + _event_pool(device_handle, context, stage::COUNT, _config), + _event{{{_event_pool.handle(), stage::UPLOAD, _config}, + {_event_pool.handle(), stage::EXECUTE, _config}, + {_event_pool.handle(), stage::READBACK, _config}}} { + const ZeroExecutor* executor = static_cast(executorPtr.get()); + static const std::size_t alignment = STANDARD_PAGE_SIZE; + + OV_ITT_SCOPED_TASK(itt::domains::LevelZeroBackend, "Zero_infer_request::DiscretePipeline::DiscretePipeline"); + for (const auto& desc : executor->inputs_desc_map()) { + _deviceInputs.appendArgument(desc.first, desc.second.info); + } + _deviceInputs.allocate(device_handle, context); + + for (const auto& desc : executor->inputs_desc_map()) { + const std::shared_ptr& inputTensor = tensors.at(desc.first); + const void* tensorBuffer = reinterpret_cast(inputTensor->data()); + + const std::size_t argSize = zeroUtils::getSizeIOBytes(desc.second.info); + std::size_t size = argSize + alignment - (argSize % alignment); + + _command_list[stage::UPLOAD].appendMemoryCopy(_deviceInputs.getDevicePtr(desc.first), tensorBuffer, size); + + executor->setArgumentValue(desc.second.idx, _deviceInputs.getDevicePtr(desc.first)); + } + + _command_list[stage::UPLOAD].appendBarrier(); + _event[stage::UPLOAD].AppendSignalEvent(_command_list[stage::UPLOAD]); + + for (const auto& desc : executor->outputs_desc_map()) { + _deviceOutputs.appendArgument(desc.first, desc.second.info); + } + _deviceOutputs.allocate(device_handle, context); + + for (const auto& desc : executor->outputs_desc_map()) { + const std::shared_ptr& outputTensor = tensors.at(desc.first); + void* tensorBuffer = reinterpret_cast(outputTensor->data()); + + const std::size_t argSize = zeroUtils::getSizeIOBytes(desc.second.info); + std::size_t size = argSize + alignment - (argSize % alignment); + + _command_list[stage::READBACK].appendMemoryCopy(tensorBuffer, + _deviceOutputs.getDevicePtr(desc.first), + size); + + executor->setArgumentValue(desc.second.idx, _deviceOutputs.getDevicePtr(desc.first)); + } + + _event[stage::UPLOAD].AppendWaitOnEvent(_command_list[stage::EXECUTE]); + + _command_list[stage::EXECUTE].appendGraphExecute(executor->graph(), profiling_handle); + + _event[stage::UPLOAD].AppendEventReset(_command_list[stage::READBACK]); + + for (auto& commandList : _command_list) { + commandList.close(); + } + }; + + DiscretePipeline(const DiscretePipeline&) = delete; + DiscretePipeline& operator=(const DiscretePipeline&) = delete; + virtual ~DiscretePipeline() = default; + + void push() override { + OV_ITT_TASK_CHAIN(ZERO_INFER_REQUEST_DP_PUSH, + itt::domains::LevelZeroBackend, + "DiscretePipeline::push", + "UPLOAD"); + // Dispatch command to copy input data from upload heap to default heap + _command_queues[stage::UPLOAD]->executeCommandList(_command_list[stage::UPLOAD]); + + OV_ITT_TASK_NEXT(ZERO_INFER_REQUEST_DP_PUSH, "EXECUTE"); + // Submit the command list for execute + _command_queues[stage::EXECUTE]->executeCommandList(_command_list[stage::EXECUTE], _fence[stage::EXECUTE]); + }; + + void pull() override { + OV_ITT_TASK_CHAIN(ZERO_INFER_REQUEST_DP_PULL, + itt::domains::LevelZeroBackend, + "DiscretePipeline::pull", + "EXECUTE"); + // Wait for execute to finish + _fence[stage::EXECUTE].hostSynchronize(); + OV_ITT_TASK_NEXT(ZERO_INFER_REQUEST_DP_PULL, "READBACK"); + // Schedule the copy of outputs from zeDriverAllocDeviceMem to zeDriverAllocHostMem + _command_queues[stage::READBACK]->executeCommandList(_command_list[stage::READBACK], _fence[stage::READBACK]); + // Wait for output copy to finish execution for _fence from the host, to make sure that data + // is available in the hostMem buffer of the output + _fence[stage::READBACK].hostSynchronize(); + }; + + void reset() const override { + // Reset the fence objects + for (auto& fence : _fence) { + fence.reset(); + } + }; + +private: + const Config _config; + const std::array, stage::COUNT>& _command_queues; + std::array _command_list; + std::array _fence; + EventPool _event_pool; + std::array _event; +}; + +struct IntegratedPipeline final : public Pipeline { +public: + IntegratedPipeline(const Config& config, + const ze_device_handle_t& device_handle, + ze_context_handle_t& context, + ze_graph_dditable_ext_curr_t* graph_ddi_table_ext, + const std::shared_ptr& executorPtr, + ze_graph_profiling_query_handle_t profiling_handle, + std::shared_ptr npu_profiling, + CommandQueue& command_queue, + const uint32_t& group_ordinal, + std::unordered_map>& tensors) + : _config(config), + _command_queue{command_queue}, + _command_list{device_handle, context, graph_ddi_table_ext, _config, group_ordinal}, + _fence{_command_queue, _config}, + _event_pool{device_handle, context, 1, _config}, + _event{_event_pool.handle(), 0, _config}, + _npu_profiling(npu_profiling) { + const ZeroExecutor* executor = static_cast(executorPtr.get()); + + OV_ITT_SCOPED_TASK(itt::domains::LevelZeroBackend, + "Zero_infer_request::IntegratedPipeline::IntegratedPipeline"); + + for (const auto& desc : executor->inputs_desc_map()) { + const std::shared_ptr& inputTensor = tensors.at(desc.first); + executor->setArgumentValue(desc.second.idx, inputTensor->data()); + } + + for (const auto& desc : executor->outputs_desc_map()) { + const std::shared_ptr& outputTensor = tensors.at(desc.first); + executor->setArgumentValue(desc.second.idx, outputTensor->data()); + } + + /// append timestamp command if feature was activated + if (_npu_profiling != nullptr) { + _command_list.appendBarrier(); + _command_list.appendNpuTimestamp(reinterpret_cast(_npu_profiling->npu_ts_infer_start)); + } + + _command_list.appendGraphExecute(executor->graph(), profiling_handle); + + /// append timestamp command if feature was activated + if (_npu_profiling != nullptr) { + _command_list.appendBarrier(); + _command_list.appendNpuTimestamp(reinterpret_cast(_npu_profiling->npu_ts_infer_end)); + } + + // appendBarrier used in L0 as well + if (!sync_output_with_fences_) { + _command_list.appendBarrier(); + _event.AppendSignalEvent(_command_list); + } + _command_list.close(); + } + + IntegratedPipeline(const IntegratedPipeline&) = delete; + IntegratedPipeline& operator=(const IntegratedPipeline&) = delete; + virtual ~IntegratedPipeline() = default; + + void push() override { + OV_ITT_TASK_CHAIN(ZERO_EXECUTOR_IP_PUSH, itt::domains::LevelZeroBackend, "IntegratedPipeline", "push"); + if (sync_output_with_fences_) { + _command_queue.executeCommandList(_command_list, _fence); + } else { + _command_queue.executeCommandList(_command_list); + } + }; + + void pull() override { + OV_ITT_TASK_CHAIN(ZERO_EXECUTOR_IP_PULL, itt::domains::LevelZeroBackend, "IntegratedPipeline", "pull"); + if (sync_output_with_fences_) { + _fence.hostSynchronize(); + } else { + _event.hostSynchronize(); + } + /// sample npu timestamps if feature was activated + if (_npu_profiling != nullptr) { + _npu_profiling->sampleNpuTimestamps(); + } + }; + + void reset() const override { + if (sync_output_with_fences_) { + _fence.reset(); + } else { + _event.reset(); + } + }; + +private: + const Config _config; + CommandQueue& _command_queue; + CommandList _command_list; + Fence _fence; + EventPool _event_pool; + Event _event; + bool sync_output_with_fences_ = true; + std::shared_ptr _npu_profiling; +}; + +std::unique_ptr makePipeline(const std::shared_ptr& executorPtr, + const Config& config, + zeroProfiling::ProfilingPool& profiling_pool, + zeroProfiling::ProfilingQuery& profiling_query, + std::shared_ptr npu_profiling, + std::unordered_map>& tensors) { + OV_ITT_SCOPED_TASK(itt::domains::LevelZeroBackend, "Infer_request::makePipeline"); + if (profiling_pool.create()) + profiling_query.create(profiling_pool._handle); + + const ZeroExecutor* executor = static_cast(executorPtr.get()); + + const ze_device_handle_t device_handle = executor->getInitStructs()->getDevice(); + ze_context_handle_t context = executor->getInitStructs()->getContext(); + ze_graph_dditable_ext_curr_t* graph_ddi_table_ext = executor->getInitStructs()->getGraphDdiTable(); + auto& command_queues = executor->getCommandQueue(); + uint32_t group_ordinal = executor->get_group_ordinal(); + + ze_device_properties_t properties = {}; + properties.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; + zeroUtils::throwOnFail("zeDeviceGetProperties", zeDeviceGetProperties(device_handle, &properties)); + + if (properties.flags & ZE_DEVICE_PROPERTY_FLAG_INTEGRATED) { + return std::make_unique(config, + device_handle, + context, + graph_ddi_table_ext, + executorPtr, + profiling_query.getHandle(), + npu_profiling, + *command_queues[stage::EXECUTE], + group_ordinal, + tensors); + } + + return std::make_unique(config, + device_handle, + context, + graph_ddi_table_ext, + executorPtr, + profiling_query.getHandle(), + command_queues, + group_ordinal, + tensors); +} + +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/backend/src/zero_profiling.cpp b/src/plugins/intel_npu/src/backend/src/zero_profiling.cpp new file mode 100644 index 00000000000..3e62ef797b5 --- /dev/null +++ b/src/plugins/intel_npu/src/backend/src/zero_profiling.cpp @@ -0,0 +1,249 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "zero_profiling.hpp" + +#include +#include + +#include "intel_npu/al/config/compiler.hpp" +#include "intel_npu/al/profiling.hpp" +#include "zero_profiling.hpp" +#include "zero_utils.hpp" + +namespace intel_npu { +namespace zeroProfiling { + +/// @brief Type trait mapping from ZE data type to enum value +template +struct ZeProfilingTypeId {}; + +template <> +struct ZeProfilingTypeId { + static const ze_graph_profiling_type_t value = ZE_GRAPH_PROFILING_LAYER_LEVEL; +}; + +template <> +struct ZeProfilingTypeId { + static const ze_graph_profiling_type_t value = ZE_GRAPH_PROFILING_TASK_LEVEL; +}; + +template <> +struct ZeProfilingTypeId { + static const ze_graph_profiling_type_t value = ZE_GRAPH_PROFILING_RAW; +}; + +bool ProfilingPool::create() { + auto ret = _graph_profiling_ddi_table_ext->pfnProfilingPoolCreate(_graph_handle, _profiling_count, &_handle); + return ((ZE_RESULT_SUCCESS == ret) && (_handle != nullptr)); +} + +ProfilingPool::~ProfilingPool() { + if (_handle) { + _graph_profiling_ddi_table_ext->pfnProfilingPoolDestroy(_handle); + } +} + +void ProfilingQuery::create(const ze_graph_profiling_pool_handle_t& profiling_pool) { + zeroUtils::throwOnFail("pfnProfilingQueryCreate", + _graph_profiling_ddi_table_ext->pfnProfilingQueryCreate(profiling_pool, _index, &_handle)); +} + +LayerStatistics ProfilingQuery::getLayerStatistics() const { + verifyProfilingProperties(); + auto layerData = getData(); + return profiling::convertLayersToIeProfilingInfo(layerData); +} + +ProfilingQuery::~ProfilingQuery() { + if (_handle) { + _graph_profiling_ddi_table_ext->pfnProfilingQueryDestroy(_handle); + } +} + +void ProfilingQuery::queryGetData(const ze_graph_profiling_type_t profilingType, + uint32_t* pSize, + uint8_t* pData) const { + if (_handle && pSize) { + zeroUtils::throwOnFail( + "pfnProfilingQueryGetData", + _graph_profiling_ddi_table_ext->pfnProfilingQueryGetData(_handle, profilingType, pSize, pData)); + } +} + +template +std::vector ProfilingQuery::getData() const { + ze_graph_profiling_type_t type = ZeProfilingTypeId::value; + uint32_t size = 0; + + // Obtain the size of the buffer + queryGetData(type, &size, nullptr); + + OPENVINO_ASSERT(size % sizeof(ProfilingData) == 0); + + // Allocate enough memory and copy the buffer + std::vector profilingData(size / sizeof(ProfilingData)); + queryGetData(type, &size, reinterpret_cast(profilingData.data())); + return profilingData; +} + +template std::vector ProfilingQuery::getData() const; + +void ProfilingQuery::getProfilingProperties(ze_device_profiling_data_properties_t* properties) const { + if (_handle && properties) { + zeroUtils::throwOnFail( + "getProfilingProperties", + _graph_profiling_ddi_table_ext->pfnDeviceGetProfilingDataProperties(_device_handle, properties)); + } +} + +void ProfilingQuery::verifyProfilingProperties() const { + if (!_handle) { + OPENVINO_THROW("Can't get profiling statistics because profiling is disabled."); + } + const auto stringifyVersion = [](auto version) -> std::string { + return std::to_string(ZE_MAJOR_VERSION(version)) + "." + std::to_string(ZE_MINOR_VERSION(version)); + }; + + ze_device_profiling_data_properties_t profProp; + getProfilingProperties(&profProp); + const auto currentProfilingVersion = ze_profiling_data_ext_version_t::ZE_PROFILING_DATA_EXT_VERSION_CURRENT; + + if (ZE_MAJOR_VERSION(profProp.extensionVersion) != ZE_MAJOR_VERSION(currentProfilingVersion)) { + OPENVINO_THROW("Unsupported NPU driver.", + "Profiling API version: plugin: ", + stringifyVersion(currentProfilingVersion), + ", driver: ", + stringifyVersion(profProp.extensionVersion)); + } + if (currentProfilingVersion > profProp.extensionVersion) { + auto log = Logger::global().clone("ZeroProfilingQuery"); + log.warning("Outdated NPU driver detected. Some features might not be available! " + "Profiling API version: plugin: %s, driver: %s", + stringifyVersion(currentProfilingVersion).c_str(), + stringifyVersion(profProp.extensionVersion).c_str()); + } +} + +NpuInferStatistics NpuInferProfiling::getNpuInferStatistics() const { + NpuInferStatistics npuPerfCounts; + + /// if the log isn't full/rolled over yet = skip reporting empty logs + uint32_t stat_cnt = (_npu_infer_stats_cnt < _npu_infer_log_maxsize) ? _npu_infer_stats_cnt : _npu_infer_log_maxsize; + if (stat_cnt != 0 && _loglevel >= ov::log::Level::WARNING) { + /// Populate npuinferstatistics vector + for (unsigned i = 0; i < stat_cnt; i++) { + ov::ProfilingInfo info; + + info.status = ov::ProfilingInfo::Status::EXECUTED; + info.real_time = std::chrono::microseconds(convertCCtoUS(_npu_infer_duration_log[i])); + info.cpu_time = std::chrono::microseconds(convertCCtoUS(_npu_infer_duration_log[i])); + info.node_name = std::to_string(i); + info.exec_type = "INFER_REQ"; + info.node_type = "INFER_REQ"; + + npuPerfCounts.push_back(info); + } + } + + /// sanity check to avoid division by 0 + if (_npu_infer_stats_cnt == 0) { + return {}; + } + + /// Add final statistics + ov::ProfilingInfo info_avg = { + ov::ProfilingInfo::Status::EXECUTED, + std::chrono::microseconds(convertCCtoUS(_npu_infer_stats_accu_cc / _npu_infer_stats_cnt)), + std::chrono::microseconds(convertCCtoUS(_npu_infer_stats_accu_cc / _npu_infer_stats_cnt)), + "AVG", + "AVG", + "AVG"}; + npuPerfCounts.push_back(info_avg); + ov::ProfilingInfo info_min = {ov::ProfilingInfo::Status::EXECUTED, + std::chrono::microseconds(convertCCtoUS(_npu_infer_stats_min_cc)), + std::chrono::microseconds(convertCCtoUS(_npu_infer_stats_min_cc)), + "MIN", + "MIN", + "MIN"}; + npuPerfCounts.push_back(info_min); + ov::ProfilingInfo info_max = {ov::ProfilingInfo::Status::EXECUTED, + std::chrono::microseconds(convertCCtoUS(_npu_infer_stats_max_cc)), + std::chrono::microseconds(convertCCtoUS(_npu_infer_stats_max_cc)), + "MAX", + "MAX", + "MAX"}; + npuPerfCounts.push_back(info_max); + return npuPerfCounts; +} + +NpuInferProfiling::NpuInferProfiling(ze_context_handle_t context, + ze_device_handle_t device_handle, + ov::log::Level loglevel) + : _context(context), + _device_handle(device_handle), + _loglevel(loglevel), + _logger("InferProfiling", loglevel) { + /// Fetch and store the device timer resolution + _dev_properties.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES_1_2; + zeroUtils::throwOnFail("zeDeviceGetProperties", zeDeviceGetProperties(_device_handle, &_dev_properties)); + /// Request mem allocations + ze_host_mem_alloc_desc_t desc = {ZE_STRUCTURE_TYPE_HOST_MEM_ALLOC_DESC, + nullptr, + ZE_HOST_MEM_ALLOC_FLAG_BIAS_CACHED}; + zeroUtils::throwOnFail("zeMemAllocHost", + zeMemAllocHost(_context, + &desc, + sizeof(uint64_t), + 64, + &npu_ts_infer_start)); // align to 64 bytes to match npu l2 cache line size + zeroUtils::throwOnFail("zeMemAllocHost", + zeMemAllocHost(_context, + &desc, + sizeof(uint64_t), + 64, + &npu_ts_infer_end)); // alight to 64 bytes to match npu l2 cache line size +} + +void NpuInferProfiling::sampleNpuTimestamps() { + int64_t infer_duration_cc = static_cast(*(reinterpret_cast(npu_ts_infer_end)) - + *(reinterpret_cast(npu_ts_infer_start))); + + /// Update extremas + if (infer_duration_cc < _npu_infer_stats_min_cc) + _npu_infer_stats_min_cc = infer_duration_cc; + if (infer_duration_cc > _npu_infer_stats_max_cc) + _npu_infer_stats_max_cc = infer_duration_cc; + _npu_infer_stats_accu_cc += infer_duration_cc; + _npu_infer_stats_cnt++; + /// only log individual infer durations if requested + if (_loglevel >= ov::log::Level::WARNING) { + _npu_infer_duration_log[_npu_infer_logidx++] = infer_duration_cc; + if (_npu_infer_logidx >= _npu_infer_log_maxsize) + _npu_infer_logidx = 0; + } +} + +int64_t NpuInferProfiling::convertCCtoUS(int64_t val_cc) const { + return (int64_t)(val_cc * 1000 * 1000 / _dev_properties.timerResolution); +} + +NpuInferProfiling::~NpuInferProfiling() { + /// deallocate npu_ts_infer_start and npu_ts_infer_end, allocated externally by ze driver + if (npu_ts_infer_start != nullptr) { + auto ze_ret = zeMemFree(_context, npu_ts_infer_start); + if (ZE_RESULT_SUCCESS != ze_ret) { + _logger.error("zeMemFree on npu_ts_infer_start failed %#X", uint64_t(ze_ret)); + } + } + if (npu_ts_infer_end != nullptr) { + auto ze_ret = zeMemFree(_context, npu_ts_infer_end); + if (ZE_RESULT_SUCCESS != ze_ret) { + _logger.error("zeMemFree on npu_ts_infer_end failed %#X", uint64_t(ze_ret)); + } + } +} + +} // namespace zeroProfiling +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/backend/src/zero_wrappers.cpp b/src/plugins/intel_npu/src/backend/src/zero_wrappers.cpp new file mode 100644 index 00000000000..85a3163ead8 --- /dev/null +++ b/src/plugins/intel_npu/src/backend/src/zero_wrappers.cpp @@ -0,0 +1,150 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "zero_wrappers.hpp" + +#include "intel_npu/al/config/common.hpp" + +namespace intel_npu { + +EventPool::EventPool(ze_device_handle_t device_handle, + const ze_context_handle_t& context, + uint32_t event_count, + const Config& config) + : _log("EventPool", config.get()) { + ze_event_pool_desc_t event_pool_desc = {ZE_STRUCTURE_TYPE_EVENT_POOL_DESC, + nullptr, + ZE_EVENT_POOL_FLAG_HOST_VISIBLE, + event_count}; + zeroUtils::throwOnFail("zeEventPoolCreate", + zeEventPoolCreate(context, &event_pool_desc, 1, &device_handle, &_handle)); +} +EventPool::~EventPool() { + auto result = zeEventPoolDestroy(_handle); + if (ZE_RESULT_SUCCESS != result) { + _log.error("zeEventPoolDestroy failed %#X", uint64_t(result)); + } +} + +Event::Event(const ze_event_pool_handle_t& event_pool, uint32_t event_index, const Config& config) + : _log("Event", config.get()) { + ze_event_desc_t event_desc = {ZE_STRUCTURE_TYPE_EVENT_DESC, nullptr, event_index, 0, 0}; + zeroUtils::throwOnFail("zeEventCreate", zeEventCreate(event_pool, &event_desc, &_handle)); +} +void Event::AppendSignalEvent(CommandList& command_list) const { + zeroUtils::throwOnFail("zeCommandListAppendSignalEvent", + zeCommandListAppendSignalEvent(command_list.handle(), _handle)); +} +void Event::AppendWaitOnEvent(CommandList& command_list) { + zeroUtils::throwOnFail("zeCommandListAppendWaitOnEvents", + zeCommandListAppendWaitOnEvents(command_list.handle(), 1, &_handle)); +} +void Event::AppendEventReset(CommandList& command_list) const { + zeroUtils::throwOnFail("zeCommandListAppendEventReset", + zeCommandListAppendEventReset(command_list.handle(), _handle)); +} +void Event::hostSynchronize() const { + zeroUtils::throwOnFail("zeEventHostSynchronize", zeEventHostSynchronize(_handle, UINT64_MAX)); +} +void Event::reset() const { + zeroUtils::throwOnFail("zeEventHostReset", zeEventHostReset(_handle)); +} +Event::~Event() { + auto result = zeEventDestroy(_handle); + if (ZE_RESULT_SUCCESS != result) { + _log.error("zeEventDestroy failed %#X", uint64_t(result)); + } +} + +CommandList::CommandList(const ze_device_handle_t& device_handle, + const ze_context_handle_t& context, + ze_graph_dditable_ext_curr_t* graph_ddi_table_ext, + const Config& config, + const uint32_t& group_ordinal) + : _context(context), + _graph_ddi_table_ext(graph_ddi_table_ext), + _log("CommandList", config.get()) { + ze_command_list_desc_t desc = {ZE_STRUCTURE_TYPE_COMMAND_LIST_DESC, nullptr, group_ordinal, 0}; + zeroUtils::throwOnFail("zeCommandListCreate", zeCommandListCreate(_context, device_handle, &desc, &_handle)); +} +void CommandList::reset() const { + zeroUtils::throwOnFail("zeCommandListReset", zeCommandListReset(_handle)); +} +void CommandList::appendMemoryCopy(void* dst, const void* src, const std::size_t size) const { + zeroUtils::throwOnFail("zeCommandListAppendMemoryCopy", + zeCommandListAppendMemoryCopy(_handle, dst, src, size, nullptr, 0, nullptr)); +} +void CommandList::appendGraphInitialize(const ze_graph_handle_t& graph_handle) const { + zeroUtils::throwOnFail("pfnAppendGraphInitialize", + _graph_ddi_table_ext->pfnAppendGraphInitialize(_handle, graph_handle, nullptr, 0, nullptr)); +} +void CommandList::appendGraphExecute(const ze_graph_handle_t& graph_handle, + const ze_graph_profiling_query_handle_t& profiling_query_handle) const { + zeroUtils::throwOnFail( + "pfnAppendGraphExecute", + _graph_ddi_table_ext + ->pfnAppendGraphExecute(_handle, graph_handle, profiling_query_handle, nullptr, 0, nullptr)); +} +void CommandList::appendNpuTimestamp(uint64_t* timestamp_buff) const { + zeroUtils::throwOnFail("zeCommandListAppendWriteGlobalTimestamp", + zeCommandListAppendWriteGlobalTimestamp(_handle, timestamp_buff, nullptr, 0, nullptr)); +} +void CommandList::appendBarrier() const { + zeroUtils::throwOnFail("zeCommandListAppendBarrier", zeCommandListAppendBarrier(_handle, nullptr, 0, nullptr)); +} +void CommandList::close() const { + zeroUtils::throwOnFail("zeCommandListClose", zeCommandListClose(_handle)); +} +CommandList::~CommandList() { + auto result = zeCommandListDestroy(_handle); + if (ZE_RESULT_SUCCESS != result) { + _log.error("zeCommandListDestroy failed %#X", uint64_t(result)); + } +} + +CommandQueue::CommandQueue(const ze_device_handle_t& device_handle, + const ze_context_handle_t& context, + const ze_command_queue_priority_t& priority, + const Config& config, + const uint32_t& group_ordinal) + : _context(context), + _log("CommandQueue", config.get()) { + ze_command_queue_desc_t queue_desc = + {ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC, nullptr, group_ordinal, 0, 0, ZE_COMMAND_QUEUE_MODE_DEFAULT, priority}; + zeroUtils::throwOnFail("zeCommandQueueCreate", + zeCommandQueueCreate(_context, device_handle, &queue_desc, &_handle)); +} +void CommandQueue::executeCommandList(CommandList& command_list) const { + zeroUtils::throwOnFail("zeCommandQueueExecuteCommandLists", + zeCommandQueueExecuteCommandLists(_handle, 1, &command_list._handle, nullptr)); +} +void CommandQueue::executeCommandList(CommandList& command_list, Fence& fence) const { + zeroUtils::throwOnFail("zeCommandQueueExecuteCommandLists", + zeCommandQueueExecuteCommandLists(_handle, 1, &command_list._handle, fence.handle())); +} +CommandQueue::~CommandQueue() { + auto result = zeCommandQueueDestroy(_handle); + if (ZE_RESULT_SUCCESS != result) { + _log.error("zeCommandQueueDestroy failed %#X", uint64_t(result)); + } +} + +Fence::Fence(const CommandQueue& command_queue, const Config& config) : _log("Fence", config.get()) { + ze_fence_desc_t fence_desc = {ZE_STRUCTURE_TYPE_FENCE_DESC, nullptr, 0}; + zeroUtils::throwOnFail("zeFenceCreate", zeFenceCreate(command_queue.handle(), &fence_desc, &_handle)); +} +void Fence::reset() const { + zeroUtils::throwOnFail("zeFenceReset", zeFenceReset(_handle)); +} +void Fence::hostSynchronize() const { + zeroUtils::throwOnFail("zeFenceHostSynchronize", zeFenceHostSynchronize(_handle, UINT64_MAX)); +} +Fence::~Fence() { + auto result = zeFenceDestroy(_handle); + if (ZE_RESULT_SUCCESS != result) { + _log.error("zeFenceDestroy failed %#X", uint64_t(result)); + } +} + +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/compiler/CMakeLists.txt b/src/plugins/intel_npu/src/compiler/CMakeLists.txt new file mode 100644 index 00000000000..ffbbc2bba53 --- /dev/null +++ b/src/plugins/intel_npu/src/compiler/CMakeLists.txt @@ -0,0 +1,37 @@ +# Copyright (C) 2018-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +set(TARGET_NAME "openvino_npu_driver_compiler_adapter") + +file(GLOB_RECURSE SOURCES *.cpp *.hpp *.h) +source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR} FILES ${SOURCES}) + +add_library(${TARGET_NAME} STATIC ${SOURCES}) + +ov_build_target_faster(${TARGET_NAME} + PCH PRIVATE "src/precomp.hpp" +) + +set_target_properties(${TARGET_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO}) +ov_add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME}) + +target_include_directories(${TARGET_NAME} + PUBLIC + $ +) + +target_link_libraries(${TARGET_NAME} + PUBLIC + openvino::npu_al + PRIVATE + openvino_npu_zero_result_parser + openvino::runtime + openvino::runtime::dev + ze_loader +) + +# +# targets install +# +ov_install_static_lib(${TARGET_NAME} ${NPU_INTERNAL_COMPONENT}) diff --git a/src/plugins/intel_npu/src/compiler/include/graph_transformations.hpp b/src/plugins/intel_npu/src/compiler/include/graph_transformations.hpp new file mode 100644 index 00000000000..ab876c44937 --- /dev/null +++ b/src/plugins/intel_npu/src/compiler/include/graph_transformations.hpp @@ -0,0 +1,22 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once +#include + +#include "npu_driver_compiler_adapter.hpp" +#include "openvino/pass/manager.hpp" + +/** + * @brief Contain all required transformation on OpenVINO model in case for external compiler usage and + * providing forward compatibility (OV model with opset N+M, external compiler with opset N) + */ +namespace intel_npu::driverCompilerAdapter { + +/** + * @brief Serialize OpenVINO model to IR + */ +IR serializeToIR(const std::shared_ptr& model, uint32_t supportedVersionByCompiler = 7); + +} // namespace intel_npu::driverCompilerAdapter diff --git a/src/plugins/intel_npu/src/compiler/include/iexternal_compiler.hpp b/src/plugins/intel_npu/src/compiler/include/iexternal_compiler.hpp new file mode 100644 index 00000000000..ef2bdbf4a9c --- /dev/null +++ b/src/plugins/intel_npu/src/compiler/include/iexternal_compiler.hpp @@ -0,0 +1,44 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once +#include "intel_npu/al/icompiler.hpp" + +namespace intel_npu { +namespace driverCompilerAdapter { + +struct IR { + std::stringstream xml; + std::stringstream weights; +}; + +/** + * @brief Interface for external compiler + * @details Isolate external API calls from general logic + */ +class IExternalCompiler { +public: + virtual ~IExternalCompiler() = default; + + /** + * @brief Get opset supported by compiler + */ + virtual uint32_t getSupportedOpset() const = 0; + + /** + * @brief Get query result for current network + */ + virtual std::unordered_set getQueryResult(IR& irModel, const Config& config) const = 0; + + /** + * @brief Sends the serialized model and its I/O metadata to the driver for compilation. + * @return The compiled model descriptor corresponding to the previously given network. + */ + virtual NetworkDescription compileIR(const std::shared_ptr& model, + IR& irModel, + const Config& config) const = 0; + virtual NetworkMetadata parseBlob(const std::vector& blob, const Config& config) const = 0; +}; +} // namespace driverCompilerAdapter +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/compiler/include/npu_driver_compiler_adapter.hpp b/src/plugins/intel_npu/src/compiler/include/npu_driver_compiler_adapter.hpp new file mode 100644 index 00000000000..dedc1a3d78b --- /dev/null +++ b/src/plugins/intel_npu/src/compiler/include/npu_driver_compiler_adapter.hpp @@ -0,0 +1,47 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include "iexternal_compiler.hpp" +#include "intel_npu/al/icompiler.hpp" +#include "intel_npu/utils/logger/logger.hpp" + +namespace intel_npu { +namespace driverCompilerAdapter { + +/** + * @brief Adapter for Compiler in driver + * @details Wrap compiler in driver calls and do preliminary actions (like opset conversion) + */ +class LevelZeroCompilerAdapter final : public ICompiler { +public: + LevelZeroCompilerAdapter(); + + uint32_t getSupportedOpsetVersion() const override final; + + NetworkDescription compile(const std::shared_ptr& model, + const Config& config) const override final; + + ov::SupportedOpsMap query(const std::shared_ptr& model, const Config& config) const override final; + + NetworkMetadata parse(const std::vector& network, const Config& config) const override final; + + std::vector process_profiling_output(const std::vector& profData, + const std::vector& network, + const Config& config) const override final; + +private: + /** + * @brief Separate externals calls to separate class + */ + std::shared_ptr apiAdapter; + ze_driver_handle_t _driverHandle = nullptr; + mutable Logger _logger; +}; + +} // namespace driverCompilerAdapter +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/compiler/include/zero_compiler_in_driver.hpp b/src/plugins/intel_npu/src/compiler/include/zero_compiler_in_driver.hpp new file mode 100644 index 00000000000..b85c9cd2198 --- /dev/null +++ b/src/plugins/intel_npu/src/compiler/include/zero_compiler_in_driver.hpp @@ -0,0 +1,220 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include + +#include + +#include "iexternal_compiler.hpp" +#include "intel_npu/utils/logger/logger.hpp" + +namespace intel_npu { +namespace driverCompilerAdapter { + +#define NotSupportLogHandle(T) \ + (std::is_same::value || std::is_same::value || \ + std::is_same::value || std::is_same::value) + +#define NotSupportQuery(T) \ + (std::is_same::value || std::is_same::value || \ + std::is_same::value) + +// ext version == 1.3 && 1.4, support API (pfnQueryNetworkCreate, pfnQueryNetworkDestroy, +// pfnQueryNetworkGetSupportedLayers) +#define SupportAPIGraphQueryNetworkV1(T) \ + (std::is_same::value || std::is_same::value) + +// ext version >= 1.5, support API (pfnCreate2, pfnQueryNetworkCreate2, pfnQueryContextMemory) +#define SupportAPIGraphQueryNetworkV2(T) ((!NotSupportQuery(T) && !SupportAPIGraphQueryNetworkV1(T))) + +// For ext version >= 1.5, pfnCreate2 api is avaible +#define NotSupportGraph2(T) \ + (std::is_same::value || std::is_same::value || \ + std::is_same::value || std::is_same::value || \ + std::is_same::value) + +/** + * Adapter to use CiD through ZeroAPI + */ +template +class LevelZeroCompilerInDriver final : public IExternalCompiler { +public: + LevelZeroCompilerInDriver(const char* extName, ze_driver_handle_t driverHandle); + LevelZeroCompilerInDriver(const LevelZeroCompilerInDriver&) = delete; + LevelZeroCompilerInDriver& operator=(const LevelZeroCompilerInDriver&) = delete; + ~LevelZeroCompilerInDriver() override; + + uint32_t getSupportedOpset() const override; + + std::unordered_set getQueryResult(IR& irModel, const Config& config) const override; + + NetworkDescription compileIR(const std::shared_ptr& model, + IR& irModel, + const Config& config) const override final; + + NetworkMetadata parseBlob(const std::vector& blob, const Config& config) const override final; + + template = true> + std::unordered_set getQueryResultFromSupportedLayers( + ze_result_t result, + ze_graph_query_network_handle_t& hGraphQueryNetwork) const; + + std::vector getSerializedIR(std::string& buildFlags, IR& irModel, const Config& config) const; + + /** + * @brief Serialize input / output information to string format. + * @details Format: + * --inputs_precisions=": [:]" + * --inputs_layouts=": [:]" + * --outputs_precisions=":" + * --outputs_layouts=":" + * + * Since the layout information is no longer an important part of the metadata values when using the 2.0 OV API, the + * layout fields shall be filled with default values in order to assure the backward compatibility with the driver. + */ + static std::string serializeIOInfo(const std::shared_ptr& model); + +private: + NetworkMetadata getNetworkMeta(ze_graph_handle_t graphHandle) const; + + std::vector serializeIR(IR& irModel, ze_graph_compiler_version_info_t compilerVersion) const; + std::string serializeConfig(const Config& config, ze_graph_compiler_version_info_t& compilerVersion) const; + + /** + * @brief Extracts the layout value or the state descriptor from the given Level Zero structure. + * @details Extracting the layout information is required only when using older driver versions which rely on this + * legacy attribute. Since this information is not found within the parameter/result nodes, we need to extract this + * value here. + * + * The state variables are also not found in the previously mentioned nodes, thus if the given Level Zero parameter + * corresponds to an input/output, we shall extract the layout value from it. Else it represents a state variable + * and the descriptor will be extracted and stored in an OpenVINO specific format. + * @param parameters Holds the already extracted input node descriptors. The transposed shape attribute of the + * corresponding entry may be updated according to the extracted layout value. + * @param results Holds the already extracted output node descriptors. The transposed shape attribute of the + * corresponding entry may be updated according to the extracted layout value. + * @param states The state descriptors shall be stored here in an OpenVINO specific format. + * @param stateNames The output location of the state variables' names in the order found within the compiled model. + * @param arg The Level Zero specific structure from which the layout value or state variable descriptor shall be + * extracted. + */ + template + void getLayoutOrStateDescriptor(IONodeDescriptorMap& parameters, + IONodeDescriptorMap& results, + IONodeDescriptorMap& states, + std::vector& stateNames, + const T& arg) const; + + /** + * @brief Extracts the node or state descriptor from the given Level Zero structure. + * @details The first version of the "ze_graph_dditable" extension does not directly support extracting the + * information corresponding to the parameter/result nodes. However, the same information can be extracted from the + * legacy I/O structures, with the exception of the node and tensor names fields. + * + * Thus, the current function is a best effort method of assuring the backward compatibility with the driver. + * The missing names have been replaced with the only one available in the structure (i.e. "arg.name"), this + * could potentially lead to issues further in the application's flow. + * @param parameters The parameter node descriptors shall be stored here in an OpenVINO specific format. + * @param results The result node descriptors shall be stored here in an OpenVINO specific format. + * @param states The state descriptors shall be stored here in an OpenVINO specific format. + * @param inputNames The output location of the input names in the order found within the compiled model. + * @param outputNames The output location of the output names in the order found within the compiled model. + * @param stateNames The output location of the state variables' names in the order found within the compiled model. + * @param arg The Level Zero specific structure from which the node or state variable descriptor shall be + * extracted. + */ + template ::value, bool> = true> + void getNodeOrStateDescriptorLegacy(IONodeDescriptorMap& parameters, + IONodeDescriptorMap& results, + IONodeDescriptorMap& states, + std::vector& inputNames, + std::vector& outputNames, + std::vector& stateNames, + const ze_graph_argument_properties_t& arg) const; + + void getMetadata(TableExtension* graphDdiTableExt, + ze_graph_handle_t graphHandle, + uint32_t index, + std::vector& inputNames, + std::vector& outputNames, + std::vector& stateNames, + IONodeDescriptorMap& parameters, + IONodeDescriptorMap& results, + IONodeDescriptorMap& state) const; + + // ext version >= 1.5, support API (pfnCreate2, pfnQueryNetworkCreate2, pfnQueryContextMemory) + template = true> + std::unordered_set queryImpl(IR& irModel, const Config& config) const; + + // ext version == 1.3 && 1.4, support API (pfnQueryNetworkCreate, pfnQueryNetworkDestroy, + // pfnQueryNetworkGetSupportedLayers) + template = true> + std::unordered_set queryImpl(IR& irModel, const Config& config) const; + + // For ext version < 1.3 + template = true> + std::unordered_set queryImpl(IR& irModel, const Config& config) const; + + template = true> + ze_result_t createGraph(const ze_graph_format_t& format, + const std::vector& serializedIR, + const std::string& buildFlags, + const uint32_t& flags, + ze_graph_handle_t* graph) const; + + template = true> + ze_result_t createGraph(const ze_graph_format_t& format, + const std::vector& serializedIR, + const std::string& buildFlags, + const uint32_t& flags, + ze_graph_handle_t* graph) const; + + template = true> + std::string getLatestBuildError() const; + + template = true> + std::string getLatestBuildError() const { + return ""; + } + +private: + ze_driver_handle_t _driverHandle = nullptr; + ze_device_handle_t _deviceHandle = nullptr; + ze_context_handle_t _context = nullptr; + + TableExtension* _graphDdiTableExt = nullptr; + mutable Logger _logger; +}; + +template +LevelZeroCompilerInDriver::LevelZeroCompilerInDriver(const char* extName, + ze_driver_handle_t driverHandle) + : _driverHandle(driverHandle), + _logger("LevelZeroCompilerInDriver", Logger::global().level()) { + // Load our graph extension + auto result = + zeDriverGetExtensionFunctionAddress(_driverHandle, extName, reinterpret_cast(&_graphDdiTableExt)); + + if (ZE_RESULT_SUCCESS != result) { + OPENVINO_THROW("Failed to initialize zeDriver. Error code: ", std::hex, result); + } + + uint32_t deviceCount = 1; + // Get our target device + result = zeDeviceGet(_driverHandle, &deviceCount, &_deviceHandle); + if (ZE_RESULT_SUCCESS != result) { + OPENVINO_THROW("Failed to get device. Error code: ", std::hex, result); + } + + ze_context_desc_t contextDesc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0}; + result = zeContextCreate(_driverHandle, &contextDesc, &_context); + if (ZE_RESULT_SUCCESS != result) { + OPENVINO_THROW("Failed to initialize context for device. Error code: ", std::hex, result); + } +} + +} // namespace driverCompilerAdapter +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/compiler/src/graph_transformations.cpp b/src/plugins/intel_npu/src/compiler/src/graph_transformations.cpp new file mode 100644 index 00000000000..40da24ff5b5 --- /dev/null +++ b/src/plugins/intel_npu/src/compiler/src/graph_transformations.cpp @@ -0,0 +1,56 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "graph_transformations.hpp" + +#include +#include + +#include "openvino/pass/serialize.hpp" +#include "transformations/op_conversions/convert_interpolate11_downgrade.hpp" + +namespace intel_npu::driverCompilerAdapter { + +IR serializeToIR(const std::shared_ptr& origModel, uint32_t supportedOpset) { + // There is no const variant of run_passes so use const_cast here + // as model serialization does not mutate the model + std::shared_ptr model = std::const_pointer_cast(origModel); + + const auto passConfig = std::make_shared(); + ov::pass::Manager manager(passConfig); + + if (supportedOpset < 11) { + // Need to clone to modify the model and remain thread safe + model = model->clone(); + // Downgrade to opset10 + manager.register_pass(); + } + + std::stringstream xmlStream, weightsStream; + manager.register_pass(xmlStream, weightsStream); + + // Depending on the driver version, the compiler attached to it may request this information as an indicator of the + // precision/layout preprocessing requirement. We are setting this value to "true" since the API version is no + // longer a cause for altering the metadata. This is due to the preprocessing performed in the OpenVINO framework's + // implementaion, the "ov::Model" object is preprocessed before reaching the NPU plugin. + const auto new_api_key = "is_new_api"; + + // We modify the original model object here therefore a mutex is required + static std::mutex rtInfoMutex; + + { + std::lock_guard lock(rtInfoMutex); + + model->set_rt_info(true, new_api_key); + + manager.run_passes(model); + + auto& rtInfo = model->get_rt_info(); + rtInfo.erase(new_api_key); + } + + return {std::move(xmlStream), std::move(weightsStream)}; +} + +} // namespace intel_npu::driverCompilerAdapter diff --git a/src/plugins/intel_npu/src/compiler/src/npu_driver_compiler_adapter.cpp b/src/plugins/intel_npu/src/compiler/src/npu_driver_compiler_adapter.cpp new file mode 100644 index 00000000000..09e9911eb25 --- /dev/null +++ b/src/plugins/intel_npu/src/compiler/src/npu_driver_compiler_adapter.cpp @@ -0,0 +1,233 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "npu_driver_compiler_adapter.hpp" + +#include "graph_transformations.hpp" +#include "intel_npu/al/config/common.hpp" +#include "intel_npu/utils/zero/zero_result.hpp" +#include "ze_intel_vpu_uuid.h" +#include "zero_compiler_in_driver.hpp" + +namespace intel_npu { +namespace driverCompilerAdapter { + +LevelZeroCompilerAdapter::LevelZeroCompilerAdapter() : _logger("LevelZeroCompilerAdapter", Logger::global().level()) { + _logger.debug("initialize zeAPI"); + auto result = zeInit(ZE_INIT_FLAG_VPU_ONLY); + if (ZE_RESULT_SUCCESS != result) { + OPENVINO_THROW("L0 initialize zeAPI", + " result: ", + ze_result_to_string(result), + ", code 0x", + std::hex, + uint64_t(result)); + } + uint32_t drivers = 0; + result = zeDriverGet(&drivers, nullptr); + + if (ZE_RESULT_SUCCESS != result) { + OPENVINO_THROW("L0 zeDriverGet get count", + " result: ", + ze_result_to_string(result), + ", code 0x", + std::hex, + uint64_t(result)); + } + + std::vector allDrivers(drivers); + result = zeDriverGet(&drivers, allDrivers.data()); + if (ZE_RESULT_SUCCESS != result) { + OPENVINO_THROW("L0 zeDriverGet get drivers", + " result: ", + ze_result_to_string(result), + ", code 0x", + std::hex, + uint64_t(result)); + } + + const ze_driver_uuid_t uuid = ze_intel_vpu_driver_uuid; + ze_driver_properties_t props = {}; + props.stype = ZE_STRUCTURE_TYPE_DRIVER_PROPERTIES; + // Get our target driver + for (uint32_t i = 0; i < drivers; ++i) { + result = zeDriverGetProperties(allDrivers[i], &props); + if (ZE_RESULT_SUCCESS != result) { + OPENVINO_THROW("L0 zeDriverGetProperties", + " result: ", + ze_result_to_string(result), + ", code 0x", + std::hex, + uint64_t(result)); + } + if (memcmp(&props.uuid, &uuid, sizeof(uuid)) == 0) { + _driverHandle = allDrivers[i]; + break; + } + } + + if (_driverHandle == nullptr) { + OPENVINO_THROW("LevelZeroCompilerAdapter: Failed to get properties about zeDriver"); + return; + } + + // query the extension properties + uint32_t count = 0; + result = zeDriverGetExtensionProperties(_driverHandle, &count, nullptr); + if (ZE_RESULT_SUCCESS != result) { + OPENVINO_THROW("L0 zeDriverGetExtensionProperties get count", + " result: ", + ze_result_to_string(result), + ", code 0x", + std::hex, + uint64_t(result)); + } + std::vector extProps; + extProps.resize(count); + result = zeDriverGetExtensionProperties(_driverHandle, &count, extProps.data()); + if (ZE_RESULT_SUCCESS != result) { + OPENVINO_THROW("L0 zeDriverGetExtensionProperties get properties", + " result: ", + ze_result_to_string(result), + ", code 0x", + std::hex, + uint64_t(result)); + } + const char* graphExtName = nullptr; + uint32_t targetVersion = 0; + for (uint32_t i = 0; i < count; ++i) { + auto& property = extProps[i]; + + if (strncmp(property.name, ZE_GRAPH_EXT_NAME, strlen(ZE_GRAPH_EXT_NAME)) != 0) + continue; + + // If the driver version is latest, will just use its name. + if (property.version == ZE_GRAPH_EXT_VERSION_CURRENT) { + graphExtName = property.name; + targetVersion = property.version; + break; + } + + // Use the latest version supported by the driver. + if (property.version > targetVersion) { + graphExtName = property.name; + targetVersion = property.version; + } + } + + if (graphExtName == nullptr) { + OPENVINO_THROW("LevelZeroCompilerAdapter: Failed to find Graph extension in NPU Driver"); + } + + const uint16_t adapterMajorVersion = 1; + uint16_t driverMajorVersion = ZE_MAJOR_VERSION(targetVersion); + if (adapterMajorVersion != driverMajorVersion) { + OPENVINO_THROW("LevelZeroCompilerAdapter: adapterMajorVersion: ", + adapterMajorVersion, + " and driverMajorVersion: ", + driverMajorVersion, + " mismatch!"); + } + +#if defined(NPU_PLUGIN_DEVELOPER_BUILD) + auto adapterManualConfig = std::getenv("ADAPTER_MANUAL_CONFIG"); + if (adapterManualConfig != nullptr) { + if (strcmp(adapterManualConfig, "ZE_extension_graph_1_5") == 0) { + _logger.info("With ADAPTER_MANUAL_CONFIG. Using ZE_GRAPH_EXT_VERSION_1_5"); + targetVersion = ZE_GRAPH_EXT_VERSION_1_5; + } else if (strcmp(adapterManualConfig, "ZE_extension_graph_1_4") == 0) { + _logger.info("With ADAPTER_MANUAL_CONFIG. Using ZE_GRAPH_EXT_VERSION_1_4"); + targetVersion = ZE_GRAPH_EXT_VERSION_1_4; + } else if (strcmp(adapterManualConfig, "ZE_extension_graph_1_3") == 0) { + _logger.info("With ADAPTER_MANUAL_CONFIG. Using ZE_GRAPH_EXT_VERSION_1_3"); + targetVersion = ZE_GRAPH_EXT_VERSION_1_3; + } else if (strcmp(adapterManualConfig, "ZE_extension_graph_1_2") == 0) { + _logger.info("With ADAPTER_MANUAL_CONFIG. Using ZE_GRAPH_EXT_VERSION_1_2"); + targetVersion = ZE_GRAPH_EXT_VERSION_1_2; + } else if (strcmp(adapterManualConfig, "ZE_extension_graph_1_1") == 0) { + _logger.info("With ADAPTER_MANUAL_CONFIG. Using ZE_GRAPH_EXT_VERSION_1_1"); + targetVersion = ZE_GRAPH_EXT_VERSION_1_1; + } else if (strcmp(adapterManualConfig, "ZE_extension_graph_1_0") == 0) { + _logger.info("With ADAPTER_MANUAL_CONFIG. Using ZE_GRAPH_EXT_VERSION_1_0"); + targetVersion = ZE_GRAPH_EXT_VERSION_1_0; + } else { + OPENVINO_THROW("Using unsupported ADAPTER_MANUAL_CONFIG!"); + } + } +#endif + if (ZE_GRAPH_EXT_VERSION_1_1 == targetVersion) { + _logger.info("Using ZE_GRAPH_EXT_VERSION_1_1"); + apiAdapter = + std::make_shared>(graphExtName, _driverHandle); + } else if (ZE_GRAPH_EXT_VERSION_1_2 == targetVersion) { + _logger.info("Using ZE_GRAPH_EXT_VERSION_1_2"); + apiAdapter = + std::make_shared>(graphExtName, _driverHandle); + } else if (strcmp(graphExtName, ZE_GRAPH_EXT_NAME_1_3) == 0) { + apiAdapter = + std::make_shared>(graphExtName, _driverHandle); + } else if (ZE_GRAPH_EXT_VERSION_1_4 == targetVersion) { + _logger.info("Using ZE_GRAPH_EXT_VERSION_1_4"); + apiAdapter = + std::make_shared>(graphExtName, _driverHandle); + } else if (ZE_GRAPH_EXT_VERSION_1_5 == targetVersion) { + _logger.info("Using ZE_GRAPH_EXT_VERSION_1_5"); + apiAdapter = + std::make_shared>(graphExtName, _driverHandle); + } else { + _logger.info("Using ZE_GRAPH_EXT_VERSION_1_0"); + apiAdapter = std::make_shared>(graphExtName, _driverHandle); + } +} + +uint32_t LevelZeroCompilerAdapter::getSupportedOpsetVersion() const { + return apiAdapter->getSupportedOpset(); +} + +NetworkDescription LevelZeroCompilerAdapter::compile(const std::shared_ptr& model, + const Config& config) const { + _logger.setLevel(config.get()); + _logger.debug("compileIR"); + uint32_t supportedOpset = apiAdapter->getSupportedOpset(); + + auto IR = serializeToIR(model, supportedOpset); + + return apiAdapter->compileIR(model, IR, config); +} + +ov::SupportedOpsMap LevelZeroCompilerAdapter::query(const std::shared_ptr& model, + const Config& config) const { + _logger.setLevel(config.get()); + _logger.debug("queryResult"); + ov::SupportedOpsMap result; + const std::string deviceName = "NPU"; + + auto IR = serializeToIR(model); + try { + const auto supportedLayers = apiAdapter->getQueryResult(IR, config); + for (auto&& layerName : supportedLayers) { + result.emplace(layerName, deviceName); + } + _logger.info("For given model, there are %d supported layers", supportedLayers.size()); + } catch (std::exception& e) { + OPENVINO_THROW("Fail in calling querynetwork : ", e.what()); + } + + return result; +} + +NetworkMetadata LevelZeroCompilerAdapter::parse(const std::vector& blob, const Config& config) const { + _logger.setLevel(config.get()); + _logger.debug("parseBlob"); + return apiAdapter->parseBlob(blob, config); +} + +std::vector LevelZeroCompilerAdapter::process_profiling_output(const std::vector&, + const std::vector&, + const Config&) const { + OPENVINO_THROW("Profiling post-processing is not implemented."); +} + +} // namespace driverCompilerAdapter +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/compiler/src/precomp.hpp b/src/plugins/intel_npu/src/compiler/src/precomp.hpp new file mode 100644 index 00000000000..14e33a30630 --- /dev/null +++ b/src/plugins/intel_npu/src/compiler/src/precomp.hpp @@ -0,0 +1,35 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include diff --git a/src/plugins/intel_npu/src/compiler/src/zero_compiler_in_driver.cpp b/src/plugins/intel_npu/src/compiler/src/zero_compiler_in_driver.cpp new file mode 100644 index 00000000000..6f4a4188f60 --- /dev/null +++ b/src/plugins/intel_npu/src/compiler/src/zero_compiler_in_driver.cpp @@ -0,0 +1,1181 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "zero_compiler_in_driver.hpp" + +#include +#include + +#include "intel_npu/al/config/common.hpp" +#include "intel_npu/al/config/runtime.hpp" +#include "intel_npu/al/itt.hpp" +#include "intel_npu/al/prefix.hpp" +#include "intel_npu/utils/zero/zero_result.hpp" +#include "openvino/core/model.hpp" + +namespace { + +constexpr std::string_view INPUTS_PRECISIONS_KEY = "--inputs_precisions"; +constexpr std::string_view INPUTS_LAYOUTS_KEY = "--inputs_layouts"; +constexpr std::string_view OUTPUTS_PRECISIONS_KEY = "--outputs_precisions"; +constexpr std::string_view OUTPUTS_LAYOUTS_KEY = "--outputs_layouts"; + +//